From 1a09ff62d38e377d762c7193fb970155f82b68d5 Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 29 Sep 2021 18:11:10 +0400 Subject: [PATCH 001/405] initialize refactoring in PatchRoute.java --- .../broadinstitute/dsm/route/PatchRoute.java | 62 +++++++++++-------- 1 file changed, 37 insertions(+), 25 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java b/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java index dbd54fba0..1ea9de5cd 100644 --- a/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java @@ -93,31 +93,8 @@ public Object processRequest(Request request, Response response, String userId) if (!Patch.patch(patch.getId(), patch.getUser(), nameValue, dbElement)) { return new RuntimeException("An error occurred while attempting to patch "); } - if (nameValue.getName().indexOf("question") > -1) { - UserDto userDto = new UserDao().getUserByEmail(patch.getUser()).orElseThrow(); - JSONObject jsonObject = new JSONObject(nameValue.getValue().toString()); - JSONArray questionArray = new JSONArray(jsonObject.get("questions").toString()); - boolean writeBack = false; - for (int i = 0; i < questionArray.length(); i++) { - JSONObject question = questionArray.getJSONObject(i); - if (question.optString(STATUS) != null && question.optString(STATUS).equals("sent")) { - if (question.optString("email") != null && question.optString("question") != null) { - notificationUtil.sentAbstractionExpertQuestion(userDto.getEmail().orElse(""), userDto.getName().orElse(""), question.optString("email"), - patch.getFieldName(), question.optString("question"), notificationUtil.getTemplate("DSM_ABSTRACTION_EXPERT_QUESTION")); - } - question.put(STATUS, "done"); - writeBack = true; - } - } - if (writeBack) { - jsonObject.put("questions", questionArray); - String str = jsonObject.toString(); - nameValue.setValue(str); - if (!Patch.patch(patch.getId(), patch.getUser(), nameValue, dbElement)) { - return new RuntimeException("An error occurred while attempting to patch "); - } - nameValues.add(nameValue); - } + if (hasQuestion(nameValue)) { + sendNotificationEmailAndUpdateStatus(patch, nameValues, nameValue, dbElement); } controlWorkflowByEmail(patch, nameValue, ddpInstance, profile); if (patch.getActions() != null) { @@ -333,6 +310,41 @@ else if (Patch.DDP_PARTICIPANT_ID.equals(patch.getParent())) { } } + private boolean hasQuestion(NameValue nameValue) { + return nameValue.getName().contains("question"); + } + + private void sendNotificationEmailAndUpdateStatus(Patch patch, List nameValues, NameValue nameValue, DBElement dbElement) { + UserDto userDto = new UserDao().getUserByEmail(patch.getUser()).orElseThrow(); + JSONObject jsonObject = new JSONObject(nameValue.getValue().toString()); + JSONArray questionArray = new JSONArray(jsonObject.get("questions").toString()); + boolean writeBack = false; + for (int i = 0; i < questionArray.length(); i++) { + JSONObject question = questionArray.getJSONObject(i); + if (isSent(question)) { + if (question.optString("email") != null && question.optString("question") != null) { + notificationUtil.sentAbstractionExpertQuestion(userDto.getEmail().orElse(""), userDto.getName().orElse(""), question.optString("email"), + patch.getFieldName(), question.optString("question"), notificationUtil.getTemplate("DSM_ABSTRACTION_EXPERT_QUESTION")); + } + question.put(STATUS, "done"); + writeBack = true; + } + } + if (writeBack) { + jsonObject.put("questions", questionArray); + String str = jsonObject.toString(); + nameValue.setValue(str); + if (!Patch.patch(patch.getId(), patch.getUser(), nameValue, dbElement)) { + throw new RuntimeException("An error occurred while attempting to patch "); + } + nameValues.add(nameValue); + } + } + + private boolean isSent(JSONObject question) { + return question.optString(STATUS) != null && question.optString(STATUS).equals("sent"); + } + private void controlWorkflowByEmail(Patch patch, NameValue nameValue, DDPInstance ddpInstance, ESProfile profile) { if (profile == null || nameValue.getValue() == null) { return; From 06f1a8c54b4a67bb10a7345b0c4a431f4f9fe7bd Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 30 Sep 2021 17:19:06 +0400 Subject: [PATCH 002/405] continue from ddp_participant_id in else if block --- .../model/familymember/AddFamilyMember.java | 2 +- .../participant/data/ParticipantData.java | 2 +- .../rgp/AutomaticProbandDataCreator.java | 3 +- .../broadinstitute/dsm/route/PatchRoute.java | 153 +++++++++++------- .../AddFamilyMemberRouteTest.java | 6 +- 5 files changed, 98 insertions(+), 68 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/familymember/AddFamilyMember.java b/src/main/java/org/broadinstitute/dsm/model/familymember/AddFamilyMember.java index 20c0febc6..a8164110e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/familymember/AddFamilyMember.java +++ b/src/main/java/org/broadinstitute/dsm/model/familymember/AddFamilyMember.java @@ -60,7 +60,7 @@ public long add() { protected void prepareFamilyMemberData() { FamilyMemberDetails familyMemberDetails = addFamilyMemberPayload.getData().orElseThrow(); - String fieldTypeId = studyGuid + ParticipantData.FIELD_TYPE; + String fieldTypeId = studyGuid + ParticipantData.FIELD_TYPE_PARTICIPANTS; participantData.setDdpParticipantId(ddpParticipantId); participantData.setDdpInstanceId(ddpInstanceId); participantData.setFieldTypeId(fieldTypeId); diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java b/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java index e8bf63bb5..7b123a187 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java @@ -27,7 +27,7 @@ public class ParticipantData { private static final Logger logger = LoggerFactory.getLogger(ParticipantData.class); - public static final String FIELD_TYPE = "_PARTICIPANTS"; + public static final String FIELD_TYPE_PARTICIPANTS = "_PARTICIPANTS"; public static final Gson GSON = new Gson(); private long dataId; diff --git a/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java b/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java index 4a082987e..b2f37f126 100644 --- a/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java +++ b/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java @@ -17,7 +17,6 @@ import org.broadinstitute.dsm.model.ddp.DDPActivityConstants; import org.broadinstitute.dsm.model.defaultvalues.Defaultable; import org.broadinstitute.dsm.model.elasticsearch.ESActivities; -import org.broadinstitute.dsm.model.elasticsearch.ElasticSearch; import org.broadinstitute.dsm.model.elasticsearch.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import org.broadinstitute.dsm.model.participant.data.FamilyMemberDetails; @@ -72,7 +71,7 @@ private boolean extractAndInsertProbandFromESData(DDPInstance instance, ElasticS participantData.setData( participantId, Integer.parseInt(instance.getDdpInstanceId()), - instance.getName().toUpperCase() + ParticipantData.FIELD_TYPE, + instance.getName().toUpperCase() + ParticipantData.FIELD_TYPE_PARTICIPANTS, probandDataMap ); participantData.addDefaultOptionsValueToData(columnsWithDefaultOptions); diff --git a/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java b/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java index 1ea9de5cd..1905da993 100644 --- a/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java @@ -2,7 +2,6 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import com.google.gson.JsonSyntaxException; import com.google.gson.reflect.TypeToken; import lombok.NonNull; import org.apache.commons.lang3.StringUtils; @@ -40,6 +39,7 @@ import spark.Request; import spark.Response; +import java.io.IOException; import java.util.*; import java.util.concurrent.CopyOnWriteArrayList; import static org.broadinstitute.ddp.db.TransactionWrapper.inTransaction; @@ -68,7 +68,7 @@ public PatchRoute(@NonNull NotificationUtil notificationUtil, @NonNull PatchUtil @Override public Object processRequest(Request request, Response response, String userId) throws Exception { - if (patchUtil.getColumnNameMap() == null) { + if (PatchUtil.getColumnNameMap() == null) { return new RuntimeException("ColumnNameMap is null!"); } String userIdRequest = UserUtil.getUserId(request); @@ -80,7 +80,7 @@ public Object processRequest(Request request, Response response, String userId) if (StringUtils.isNotBlank(patch.getId())) { //multiple values are changing DDPInstance ddpInstance = DDPInstance.getDDPInstance(patch.getRealm()); - if (patch.getNameValues() != null && !patch.getNameValues().isEmpty()) { + if (isNameValuePairs(patch)) { List nameValues = new ArrayList<>(); ESProfile profile = ElasticSearchUtil.getParticipantProfileByGuidOrAltPid(ddpInstance.getParticipantIndexES(), patch.getDdpParticipantId()) .orElse(null); @@ -88,7 +88,7 @@ public Object processRequest(Request request, Response response, String userId) logger.error("Unable to find ES profile for participant with guid/altpid: {}, continuing w/ patch", patch.getParentId()); } for (NameValue nameValue : patch.getNameValues()) { - DBElement dbElement = patchUtil.getColumnNameMap().get(nameValue.getName()); + DBElement dbElement = PatchUtil.getColumnNameMap().get(nameValue.getName()); if (dbElement != null) { if (!Patch.patch(patch.getId(), patch.getUser(), nameValue, dbElement)) { return new RuntimeException("An error occurred while attempting to patch "); @@ -98,14 +98,7 @@ public Object processRequest(Request request, Response response, String userId) } controlWorkflowByEmail(patch, nameValue, ddpInstance, profile); if (patch.getActions() != null) { - for (Value action : patch.getActions()) { - if (ESObjectConstants.ELASTIC_EXPORT_WORKFLOWS.equals(action.getType()) && profile != null) { - writeESWorkflow(patch, nameValue, action, ddpInstance, profile.getParticipantGuid()); - } - else if (EventTypeDao.EVENT.equals(action.getType())) { - triggerParticipantEvent(ddpInstance, patch, action); - } - } + writeESWorkflowElseTriggerParticipantEvent(patch, ddpInstance, profile, nameValue); } } else { @@ -116,7 +109,7 @@ else if (EventTypeDao.EVENT.equals(action.getType())) { } else { // mr changes - DBElement dbElement = patchUtil.getColumnNameMap().get(patch.getNameValue().getName()); + DBElement dbElement = PatchUtil.getColumnNameMap().get(patch.getNameValue().getName()); if (dbElement != null) { if (Patch.patch(patch.getId(), patch.getUser(), patch.getNameValue(), dbElement)) { List nameValues = setWorkflowRelatedFields(patch); @@ -131,14 +124,14 @@ else if (EventTypeDao.EVENT.equals(action.getType())) { } } else if (StringUtils.isNotBlank(patch.getParent()) && StringUtils.isNotBlank(patch.getParentId())) { - if (Patch.PARTICIPANT_ID.equals(patch.getParent())) { + if (Patch.PARTICIPANT_ID.equals(patch.getParent())) { //Abstraction, Medical Record, Onc history related tables changes if (StringUtils.isNotBlank(patch.getFieldId())) { //abstraction related change //multiple value - if (patch.getNameValues() != null && !patch.getNameValues().isEmpty()) { + if (isNameValuePairs(patch)) { String primaryKeyId = null; for (NameValue nameValue : patch.getNameValues()) { - DBElement dbElement = patchUtil.getColumnNameMap().get(nameValue.getName()); + DBElement dbElement = PatchUtil.getColumnNameMap().get(nameValue.getName()); if (dbElement != null) { if (primaryKeyId == null) { primaryKeyId = AbstractionWrapper.createNewAbstractionFieldValue(patch.getParentId(), patch.getFieldId(), patch.getUser(), nameValue, dbElement); @@ -158,7 +151,7 @@ else if (StringUtils.isNotBlank(patch.getParent()) && StringUtils.isNotBlank(pat } else { //single value - DBElement dbElement = patchUtil.getColumnNameMap().get(patch.getNameValue().getName()); + DBElement dbElement = PatchUtil.getColumnNameMap().get(patch.getNameValue().getName()); if (dbElement != null) { String primaryKeyId = AbstractionWrapper.createNewAbstractionFieldValue(patch.getParentId(), patch.getFieldId(), patch.getUser(), patch.getNameValue(), dbElement); Map map = new HashMap<>(); @@ -185,7 +178,7 @@ else if (StringUtils.isNotBlank(patch.getParent()) && StringUtils.isNotBlank(pat List nameValues = null; Map map = new HashMap<>(); //facility was added - if (patch.getNameValues() != null && !patch.getNameValues().isEmpty()) { + if (isNameValuePairs(patch)) { for (NameValue nameValue : patch.getNameValues()) { DBElement dbElement = patchUtil.getColumnNameMap().get(nameValue.getName()); if (dbElement != null) { @@ -230,6 +223,7 @@ else if (StringUtils.isNotBlank(patch.getParent()) && StringUtils.isNotBlank(pat } } else if (Patch.ONC_HISTORY_ID.equals(patch.getParent())) { + // ddp_tissue table related change String tissueId = Tissue.createNewTissue(patch.getParentId(), patch.getUser()); DBElement dbElement = patchUtil.getColumnNameMap().get(patch.getNameValue().getName()); if (dbElement != null) { @@ -251,6 +245,7 @@ else if (Patch.ONC_HISTORY_ID.equals(patch.getParent())) { } } else if (Patch.PARTICIPANT_DATA_ID.equals(patch.getParent())) { + //ddp_participant_data table String participantDataId = null; Map map = new HashMap<>(); DDPInstance ddpInstance = DDPInstance.getDDPInstance(patch.getRealm()); @@ -268,7 +263,7 @@ else if (participantDataId != null) { ESProfile profile = ElasticSearchUtil.getParticipantProfileByGuidOrAltPid(ddpInstance.getParticipantIndexES(), patch.getParentId()) .orElseThrow(() -> new RuntimeException("Unable to find ES profile for participant: " + patch.getParentId())); for (Value action : patch.getActions()) { - if (ESObjectConstants.ELASTIC_EXPORT_WORKFLOWS.equals(action.getType())) { + if (hasProfileAndESWorkflowType(profile, action)) { writeESWorkflow(patch, nameValue, action, ddpInstance, profile.getParticipantGuid()); } else if (EventTypeDao.EVENT.equals(action.getType())) { @@ -310,6 +305,26 @@ else if (Patch.DDP_PARTICIPANT_ID.equals(patch.getParent())) { } } + private boolean isNameValuePairs(Patch patch) { + //TODO -> could be changed later after clarification + return patch.getNameValues() != null && !patch.getNameValues().isEmpty(); + } + + private void writeESWorkflowElseTriggerParticipantEvent(Patch patch, DDPInstance ddpInstance, ESProfile profile, NameValue nameValue) { + for (Value action : patch.getActions()) { + if (hasProfileAndESWorkflowType(profile, action)) { + writeESWorkflow(patch, nameValue, action, ddpInstance, profile.getParticipantGuid()); + } + else if (EventTypeDao.EVENT.equals(action.getType())) { + triggerParticipantEvent(ddpInstance, patch, action); + } + } + } + + private boolean hasProfileAndESWorkflowType(ESProfile profile, Value action) { + return ESObjectConstants.ELASTIC_EXPORT_WORKFLOWS.equals(action.getType()) && profile != null; + } + private boolean hasQuestion(NameValue nameValue) { return nameValue.getName().contains("question"); } @@ -357,47 +372,54 @@ private void controlWorkflowByEmail(Patch patch, NameValue nameValue, DDPInstanc pData); if (participantData.hasFamilyMemberApplicantEmail(profile)) { - logger.info("Email in patch data matches participant profile email, will update workflows"); - int ddpInstanceIdByGuid = Integer.parseInt(ddpInstance.getDdpInstanceId()); - FieldSettings fieldSettings = new FieldSettings(); - pData.forEach((columnName,columnValue) -> { - if (!fieldSettings.isColumnExportable(ddpInstanceIdByGuid, columnName)) return; - if (!patch.getFieldId().contains(org.broadinstitute.dsm.model.participant.data.ParticipantData.FIELD_TYPE)) return; - // Use participant guid here to avoid multiple ES lookups. - ElasticSearchUtil.writeWorkflow(WorkflowForES.createInstanceWithStudySpecificData(ddpInstance, - profile.getParticipantGuid(), columnName, columnValue, new WorkflowForES.StudySpecificData( - pData.get(FamilyMemberConstants.COLLABORATOR_PARTICIPANT_ID), - pData.get(FamilyMemberConstants.FIRSTNAME), - pData.get(FamilyMemberConstants.LASTNAME))), false); - }); + writeFamilyMemberWorklow(patch, ddpInstance, profile, pData); } else { - logger.info("Email in patch data does not match participant profile email, will remove workflows"); Map esMap = ElasticSearchUtil .getObjectsMap(ddpInstance.getParticipantIndexES(), profile.getParticipantGuid(), ESObjectConstants.WORKFLOWS); if (Objects.isNull(esMap) || esMap.isEmpty()) return; - CopyOnWriteArrayList> workflowsList = new CopyOnWriteArrayList<>((List>)esMap.get(ESObjectConstants.WORKFLOWS)); - int startingSize = workflowsList.size(); - workflowsList.forEach(workflow -> { - Map workflowDataMap = (Map) workflow.get(ESObjectConstants.DATA); - String collaboratorParticipantId = workflowDataMap.get(ESObjectConstants.SUBJECT_ID); - if (Objects.isNull(collaboratorParticipantId)) return; - if (collaboratorParticipantId.equals(pData.get(FamilyMemberConstants.COLLABORATOR_PARTICIPANT_ID))) { - workflowsList.remove(workflow); - } - }); - if (startingSize != workflowsList.size()) { - esMap.put(ESObjectConstants.WORKFLOWS, workflowsList); - // Use participant guid here to avoid another ES lookup. - ElasticSearchUtil.updateRequest(profile.getParticipantGuid(), ddpInstance.getParticipantIndexES(), esMap); - } + removeFamilyMemberWorkflowData(ddpInstance, profile, pData, esMap); } - } catch (JsonSyntaxException ignored) { } catch (Exception e) { throw new RuntimeException(e); } } + private void removeFamilyMemberWorkflowData(DDPInstance ddpInstance, ESProfile profile, Map pData, Map esMap) throws IOException { + logger.info("Email in patch data does not match participant profile email, will remove workflows"); + CopyOnWriteArrayList> workflowsList = new CopyOnWriteArrayList<>((List>) esMap.get(ESObjectConstants.WORKFLOWS)); + int startingSize = workflowsList.size(); + workflowsList.forEach(workflow -> { + Map workflowDataMap = (Map) workflow.get(ESObjectConstants.DATA); + String collaboratorParticipantId = workflowDataMap.get(ESObjectConstants.SUBJECT_ID); + if (Objects.isNull(collaboratorParticipantId)) return; + if (collaboratorParticipantId.equals(pData.get(FamilyMemberConstants.COLLABORATOR_PARTICIPANT_ID))) { + workflowsList.remove(workflow); + } + }); + if (startingSize != workflowsList.size()) { + esMap.put(ESObjectConstants.WORKFLOWS, workflowsList); + // Use participant guid here to avoid another ES lookup. + ElasticSearchUtil.updateRequest(profile.getParticipantGuid(), ddpInstance.getParticipantIndexES(), esMap); + } + } + + private void writeFamilyMemberWorklow(Patch patch, DDPInstance ddpInstance, ESProfile profile, Map pData) { + logger.info("Email in patch data matches participant profile email, will update workflows"); + int ddpInstanceIdByGuid = Integer.parseInt(ddpInstance.getDdpInstanceId()); + FieldSettings fieldSettings = new FieldSettings(); + pData.forEach((columnName, columnValue) -> { + if (!fieldSettings.isColumnExportable(ddpInstanceIdByGuid, columnName)) return; + if (!patch.getFieldId().contains(org.broadinstitute.dsm.model.participant.data.ParticipantData.FIELD_TYPE_PARTICIPANTS)) return; + // Use participant guid here to avoid multiple ES lookups. + ElasticSearchUtil.writeWorkflow(WorkflowForES.createInstanceWithStudySpecificData(ddpInstance, + profile.getParticipantGuid(), columnName, columnValue, new WorkflowForES.StudySpecificData( + pData.get(FamilyMemberConstants.COLLABORATOR_PARTICIPANT_ID), + pData.get(FamilyMemberConstants.FIRSTNAME), + pData.get(FamilyMemberConstants.LASTNAME))), false); + }); + } + private void insertDdpParticipantRecord(int participantId) { ParticipantRecordDto participantRecordDto = new ParticipantRecordDto.Builder(participantId, System.currentTimeMillis()) @@ -420,23 +442,17 @@ private int insertDdpParticipant(Patch patch, DDPInstance ddpInstance) { private void writeDSMRecordsToES(@NonNull Patch patch, DDPInstance ddpInstance) { NameValue nameValue = patch.getNameValue(); String name = nameValue.getName().substring(nameValue.getName().lastIndexOf('.') + 1); - String type = null; - if (nameValue.getName().indexOf('.') != -1) { - type = nameValue.getName().substring(0, nameValue.getName().indexOf('.')); - } - else { - return; - } + String type = getTypeFrom(nameValue); + if (type == null) return; String value = nameValue.getValue().toString(); Map nameValueMap = new HashMap<>(); nameValueMap.put(name, value); - if (DBConstants.DDP_MEDICAL_RECORD_ALIAS.equals(type)) { - if (ESObjectConstants.MEDICAL_RECORDS_FIELD_NAMES.contains(name)) { - ElasticSearchUtil.writeDsmRecord(ddpInstance, Integer.parseInt(patch.getId()), patch.getDdpParticipantId(), - ESObjectConstants.MEDICAL_RECORDS, ESObjectConstants.MEDICAL_RECORDS_ID, nameValueMap); - } + if (isMedicalRecord(name, type)) { + ElasticSearchUtil.writeDsmRecord(ddpInstance, Integer.parseInt(patch.getId()), patch.getDdpParticipantId(), + ESObjectConstants.MEDICAL_RECORDS, ESObjectConstants.MEDICAL_RECORDS_ID, nameValueMap); } else if (DBConstants.DDP_ONC_HISTORY_DETAIL_ALIAS.equals(type)) { + // TODO relationship between DDP_ONC_HISTORY_DETAIL_ALIAS and TISSUE_RECORDS_FIELD_NAMES if (ESObjectConstants.TISSUE_RECORDS_FIELD_NAMES.contains(name)) { if (PARTICIPANT_ID.equals(patch.getParent())) { ElasticSearchUtil.writeDsmRecord(ddpInstance, Integer.parseInt(patch.getId()), patch.getDdpParticipantId(), @@ -446,6 +462,21 @@ else if (DBConstants.DDP_ONC_HISTORY_DETAIL_ALIAS.equals(type)) { } } + private String getTypeFrom(NameValue nameValue) { + String type = null; + if (nameValue.getName().indexOf('.') != -1) { + type = nameValue.getName().substring(0, nameValue.getName().indexOf('.')); + } + else { + return null; + } + return type; + } + + private boolean isMedicalRecord(String name, String type) { + return DBConstants.DDP_MEDICAL_RECORD_ALIAS.equals(type) && ESObjectConstants.MEDICAL_RECORDS_FIELD_NAMES.contains(name); + } + private void writeESWorkflow(@NonNull Patch patch, @NonNull NameValue nameValue, @NonNull Value action, DDPInstance ddpInstance, String esParticipantId) { String status = nameValue.getValue() != null ? String.valueOf(nameValue.getValue()) : null; if (StringUtils.isBlank(status)) { diff --git a/src/test/java/org/broadinstitute/dsm/route/familymember/AddFamilyMemberRouteTest.java b/src/test/java/org/broadinstitute/dsm/route/familymember/AddFamilyMemberRouteTest.java index d048e353f..e5d672374 100644 --- a/src/test/java/org/broadinstitute/dsm/route/familymember/AddFamilyMemberRouteTest.java +++ b/src/test/java/org/broadinstitute/dsm/route/familymember/AddFamilyMemberRouteTest.java @@ -80,7 +80,7 @@ private static void createProbandTestParticipantData() { new ParticipantDataDto.Builder() .withDdpParticipantId(participantId) .withDdpInstanceId(ddpInstanceDto.getDdpInstanceId()) - .withFieldTypeId(ParticipantData.FIELD_TYPE) + .withFieldTypeId(ParticipantData.FIELD_TYPE_PARTICIPANTS) .withData(gson.toJson(probandData)) .withLastChanged(System.currentTimeMillis()) .withChangedBy(userDto.getEmail().orElse("SYSTEM")) @@ -155,7 +155,7 @@ public void relationshipIdAlreadyExists() { AddFamilyMemberPayload addFamilyMemberPayload = gson.fromJson(payload, AddFamilyMemberPayload.class); ParticipantData participantData = new ParticipantData(participantDataDao); participantData.setData(addFamilyMemberPayload.getParticipantId().get(), ddpInstanceDto.getDdpInstanceId(), - ddpInstanceDto.getInstanceName() + ParticipantData.FIELD_TYPE, probandData); + ddpInstanceDto.getInstanceName() + ParticipantData.FIELD_TYPE_PARTICIPANTS, probandData); Assert.assertTrue(participantData.isRelationshipIdExists()); } @@ -169,7 +169,7 @@ public void addFamilyMemberToParticipant() { new ParticipantDataDto.Builder() .withDdpParticipantId(addFamilyMemberPayload.getParticipantId().get()) .withDdpInstanceId(ddpInstanceDto.getDdpInstanceId()) - .withFieldTypeId(ddpInstanceDto.getInstanceName() + ParticipantData.FIELD_TYPE) + .withFieldTypeId(ddpInstanceDto.getInstanceName() + ParticipantData.FIELD_TYPE_PARTICIPANTS) .withData(gson.toJson(addFamilyMemberPayload.getData().get())) .withLastChanged(System.currentTimeMillis()) .withChangedBy(userDto.getEmail().orElse("SYSTEM")) From 13ca2090b0587abe379682869a001862172a459c Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 4 Oct 2021 10:33:18 +0400 Subject: [PATCH 003/405] add skeleton for refactoring --- .../org/broadinstitute/dsm/model/patch/BasePatch.java | 6 ++++++ .../broadinstitute/dsm/model/patch/PatchFactory.java | 8 ++++++++ .../org/broadinstitute/dsm/model/patch/Patchable.java | 5 +++++ .../java/org/broadinstitute/dsm/route/PatchRoute.java | 10 +++++++--- 4 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/patch/Patchable.java diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java new file mode 100644 index 000000000..776a5f39c --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -0,0 +1,6 @@ +package org.broadinstitute.dsm.model.patch; + +public abstract class BasePatch implements Patchable { + + +} diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java new file mode 100644 index 000000000..d081e8fdb --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java @@ -0,0 +1,8 @@ +package org.broadinstitute.dsm.model.patch; + +public class PatchFactory { + + public static Patchable makePatch() { + return null; + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/Patchable.java b/src/main/java/org/broadinstitute/dsm/model/patch/Patchable.java new file mode 100644 index 000000000..75eded389 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/patch/Patchable.java @@ -0,0 +1,5 @@ +package org.broadinstitute.dsm.model.patch; + +public interface Patchable { + +} diff --git a/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java b/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java index 1905da993..feba34eae 100644 --- a/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java @@ -77,7 +77,7 @@ public Object processRequest(Request request, Response response, String userId) try { String requestBody = request.body(); Patch patch = gson.fromJson(requestBody, Patch.class); - if (StringUtils.isNotBlank(patch.getId())) { + if (hasPrimaryKey(patch)) { //multiple values are changing DDPInstance ddpInstance = DDPInstance.getDDPInstance(patch.getRealm()); if (isNameValuePairs(patch)) { @@ -125,7 +125,7 @@ public Object processRequest(Request request, Response response, String userId) } else if (StringUtils.isNotBlank(patch.getParent()) && StringUtils.isNotBlank(patch.getParentId())) { if (Patch.PARTICIPANT_ID.equals(patch.getParent())) { //Abstraction, Medical Record, Onc history related tables changes - if (StringUtils.isNotBlank(patch.getFieldId())) { + if (StringUtils.isNotBlank(patch.getFieldId())) { // medical record abstraction field id //abstraction related change //multiple value if (isNameValuePairs(patch)) { @@ -137,7 +137,7 @@ else if (StringUtils.isNotBlank(patch.getParent()) && StringUtils.isNotBlank(pat primaryKeyId = AbstractionWrapper.createNewAbstractionFieldValue(patch.getParentId(), patch.getFieldId(), patch.getUser(), nameValue, dbElement); } if (!Patch.patch(primaryKeyId, patch.getUser(), nameValue, dbElement)) { - return new RuntimeException("An error occurred while attempting to patch "); + throw new RuntimeException("An error occurred while attempting to patch "); } } else { @@ -305,6 +305,10 @@ else if (Patch.DDP_PARTICIPANT_ID.equals(patch.getParent())) { } } + private boolean hasPrimaryKey(Patch patch) { + return StringUtils.isNotBlank(patch.getId()); + } + private boolean isNameValuePairs(Patch patch) { //TODO -> could be changed later after clarification return patch.getNameValues() != null && !patch.getNameValues().isEmpty(); From 719560773c003d76761641aaa06c87be80ce44ca Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 4 Oct 2021 11:09:06 +0400 Subject: [PATCH 004/405] Refactor | introduce null object design pattern, template method design pattern --- .../broadinstitute/dsm/model/patch/BasePatch.java | 7 +++++++ .../broadinstitute/dsm/model/patch/NullPatch.java | 14 ++++++++++++++ .../dsm/model/patch/PatchFactory.java | 11 +++++++++-- .../org/broadinstitute/dsm/route/PatchRoute.java | 3 ++- 4 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index 776a5f39c..87aea86f3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -1,6 +1,13 @@ package org.broadinstitute.dsm.model.patch; +import java.util.List; +import java.util.Optional; + +import org.broadinstitute.dsm.model.NameValue; + public abstract class BasePatch implements Patchable { + abstract Optional processMultipleNameValues(List nameValues); + } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java new file mode 100644 index 000000000..2ac2eddc1 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java @@ -0,0 +1,14 @@ +package org.broadinstitute.dsm.model.patch; + +import java.util.List; +import java.util.Optional; + +import org.broadinstitute.dsm.model.NameValue; + +public class NullPatch extends BasePatch { + + @Override + Optional processMultipleNameValues(List nameValues) { + return Optional.empty(); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java index d081e8fdb..5eb27967d 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java @@ -1,8 +1,15 @@ package org.broadinstitute.dsm.model.patch; +import org.broadinstitute.dsm.model.Patch; + public class PatchFactory { - public static Patchable makePatch() { - return null; + public static Patchable makePatch(Patch patch) { + Patchable patcher = new NullPatch(); + // switch cases here + return patcher; } + + + } diff --git a/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java b/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java index feba34eae..09c72f646 100644 --- a/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java @@ -90,8 +90,9 @@ public Object processRequest(Request request, Response response, String userId) for (NameValue nameValue : patch.getNameValues()) { DBElement dbElement = PatchUtil.getColumnNameMap().get(nameValue.getName()); if (dbElement != null) { + // basePatch.processMultipleNameValues(patch.getNameValues()) if (!Patch.patch(patch.getId(), patch.getUser(), nameValue, dbElement)) { - return new RuntimeException("An error occurred while attempting to patch "); + throw new RuntimeException("An error occurred while attempting to patch "); } if (hasQuestion(nameValue)) { sendNotificationEmailAndUpdateStatus(patch, nameValues, nameValue, dbElement); From 68ad44614a45a8dd75f5245ffbe43866a7f1190a Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 4 Oct 2021 11:34:02 +0400 Subject: [PATCH 005/405] initialize PrimaryKeyPatch.java --- .../dsm/model/patch/BasePatch.java | 37 ++++++++++++++++++- .../dsm/model/patch/NullPatch.java | 5 +++ .../dsm/model/patch/PrimaryKeyPatch.java | 31 ++++++++++++++++ 3 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index 87aea86f3..250363c50 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -3,11 +3,46 @@ import java.util.List; import java.util.Optional; +import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; +import org.broadinstitute.dsm.model.Patch; +import org.broadinstitute.dsm.util.PatchUtil; + public abstract class BasePatch implements Patchable { + protected Patch patch; + + public BasePatch() { + + } + + public BasePatch(Patch patch) { + this.patch = patch; + } + + abstract Optional processSingleNameValue(NameValue nameValue, DBElement dbElement); + + Optional processMultipleNameValues(List nameValues) { + Optional result = Optional.empty(); + for (NameValue nameValue : patch.getNameValues()) { + DBElement dbElement = PatchUtil.getColumnNameMap().get(nameValue.getName()); + if (dbElement != null) { + result = processSingleNameValue(nameValue, dbElement); + } + else { + throw new RuntimeException("DBElement not found in ColumnNameMap: " + nameValue.getName()); + } + } + return result; + } + + private boolean hasQuestion(NameValue nameValue) { + return nameValue.getName().contains("question"); + } + + + - abstract Optional processMultipleNameValues(List nameValues); } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java index 2ac2eddc1..34e9971c9 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java @@ -7,6 +7,11 @@ public class NullPatch extends BasePatch { + @Override + Optional processSingleNameValue(NameValue nameValue) { + return Optional.empty(); + } + @Override Optional processMultipleNameValues(List nameValues) { return Optional.empty(); diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java new file mode 100644 index 000000000..b442a14f4 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java @@ -0,0 +1,31 @@ +package org.broadinstitute.dsm.model.patch; + +import org.broadinstitute.dsm.db.structure.DBElement; +import org.broadinstitute.dsm.model.NameValue; +import org.broadinstitute.dsm.model.Patch; + +import java.util.Optional; + +public class PrimaryKeyPatch extends BasePatch { + + public PrimaryKeyPatch(Patch patch) { + super(patch); + } + + @Override + Optional processSingleNameValue(NameValue nameValue, DBElement dbElement) { + + if (!Patch.patch(patch.getId(), patch.getUser(), nameValue, dbElement)) { + throw new RuntimeException("An error occurred while attempting to patch "); + } + if (hasQuestion(nameValue)) { + sendNotificationEmailAndUpdateStatus(patch, nameValues, nameValue, dbElement); + } + controlWorkflowByEmail(patch, nameValue, ddpInstance, profile); + if (patch.getActions() != null) { + writeESWorkflowElseTriggerParticipantEvent(patch, ddpInstance, profile, nameValue); + } + + return Optional.empty(); + } +} From f2f378017b436c05231e6db2a40428e69a5bb409 Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 4 Oct 2021 15:36:06 +0400 Subject: [PATCH 006/405] add further modifications --- .../dsm/model/patch/BasePatch.java | 21 ++++--- .../dsm/model/patch/NullPatch.java | 3 +- .../dsm/model/patch/PrimaryKeyPatch.java | 55 +++++++++++++++++-- .../broadinstitute/dsm/route/PatchRoute.java | 2 + 4 files changed, 67 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index 250363c50..e3ab66616 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -1,48 +1,53 @@ package org.broadinstitute.dsm.model.patch; +import java.util.ArrayList; import java.util.List; import java.util.Optional; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.Patch; +import org.broadinstitute.dsm.util.NotificationUtil; import org.broadinstitute.dsm.util.PatchUtil; public abstract class BasePatch implements Patchable { + protected static final String PARTICIPANT_ID = "participantId"; + protected static final String PRIMARY_KEY_ID = "primaryKeyId"; + protected static final String NAME_VALUE = "NameValue"; + protected static final String STATUS = "status"; + protected Patch patch; public BasePatch() { - } public BasePatch(Patch patch) { this.patch = patch; } - abstract Optional processSingleNameValue(NameValue nameValue, DBElement dbElement); + abstract Optional processSingleNameValue(NameValue nameValue, DBElement dbElement); - Optional processMultipleNameValues(List nameValues) { - Optional result = Optional.empty(); + List processMultipleNameValues(List nameValues) { + List updatedNameValues = new ArrayList<>(); for (NameValue nameValue : patch.getNameValues()) { DBElement dbElement = PatchUtil.getColumnNameMap().get(nameValue.getName()); if (dbElement != null) { - result = processSingleNameValue(nameValue, dbElement); + processSingleNameValue(nameValue, dbElement).ifPresent(updatedNameValues::add); } else { throw new RuntimeException("DBElement not found in ColumnNameMap: " + nameValue.getName()); } } - return result; + return updatedNameValues; } - private boolean hasQuestion(NameValue nameValue) { + protected boolean hasQuestion(NameValue nameValue) { return nameValue.getName().contains("question"); } - } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java index 34e9971c9..d5033b4e3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java @@ -3,12 +3,13 @@ import java.util.List; import java.util.Optional; +import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; public class NullPatch extends BasePatch { @Override - Optional processSingleNameValue(NameValue nameValue) { + Optional processSingleNameValue(NameValue nameValue, DBElement dbElement) { return Optional.empty(); } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java index b442a14f4..920f767aa 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java @@ -1,31 +1,76 @@ package org.broadinstitute.dsm.model.patch; +import org.broadinstitute.dsm.db.dao.user.UserDao; +import org.broadinstitute.dsm.db.dto.user.UserDto; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.Patch; +import org.broadinstitute.dsm.util.NotificationUtil; +import org.json.JSONArray; +import org.json.JSONObject; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.Optional; public class PrimaryKeyPatch extends BasePatch { - public PrimaryKeyPatch(Patch patch) { + private NotificationUtil notificationUtil; + + public PrimaryKeyPatch(Patch patch, NotificationUtil notificationUtil) { super(patch); + this.notificationUtil = notificationUtil; } @Override - Optional processSingleNameValue(NameValue nameValue, DBElement dbElement) { - + Optional processSingleNameValue(NameValue nameValue, DBElement dbElement) { + Optional maybeUpdatedNameValue = Optional.empty(); if (!Patch.patch(patch.getId(), patch.getUser(), nameValue, dbElement)) { throw new RuntimeException("An error occurred while attempting to patch "); } if (hasQuestion(nameValue)) { - sendNotificationEmailAndUpdateStatus(patch, nameValues, nameValue, dbElement); + maybeUpdatedNameValue = sendNotificationEmailAndUpdateStatus(patch, nameValue, dbElement); } controlWorkflowByEmail(patch, nameValue, ddpInstance, profile); if (patch.getActions() != null) { writeESWorkflowElseTriggerParticipantEvent(patch, ddpInstance, profile, nameValue); } - return Optional.empty(); + return maybeUpdatedNameValue; } + + private Optional sendNotificationEmailAndUpdateStatus(Patch patch, NameValue nameValue, DBElement dbElement) { + Optional maybeUpdatedNameValue = Optional.empty(); + UserDto userDto = new UserDao().getUserByEmail(patch.getUser()).orElseThrow(); + JSONObject jsonObject = new JSONObject(nameValue.getValue().toString()); + JSONArray questionArray = new JSONArray(jsonObject.get("questions").toString()); + boolean writeBack = false; + for (int i = 0; i < questionArray.length(); i++) { + JSONObject question = questionArray.getJSONObject(i); + if (isSent(question)) { + if (question.optString("email") != null && question.optString("question") != null) { + notificationUtil.sentAbstractionExpertQuestion(userDto.getEmail().orElse(""), userDto.getName().orElse(""), question.optString("email"), + patch.getFieldName(), question.optString("question"), notificationUtil.getTemplate("DSM_ABSTRACTION_EXPERT_QUESTION")); + } + question.put(STATUS, "done"); + writeBack = true; + } + } + if (writeBack) { + jsonObject.put("questions", questionArray); + String str = jsonObject.toString(); + nameValue.setValue(str); + if (!Patch.patch(patch.getId(), patch.getUser(), nameValue, dbElement)) { + throw new RuntimeException("An error occurred while attempting to patch "); + } + maybeUpdatedNameValue = Optional.of(nameValue); + } + return maybeUpdatedNameValue; + } + + private boolean isSent(JSONObject question) { + return question.optString(STATUS) != null && question.optString(STATUS).equals("sent"); + } + } diff --git a/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java b/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java index 09c72f646..7f7cef71b 100644 --- a/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java @@ -87,6 +87,8 @@ public Object processRequest(Request request, Response response, String userId) if (profile == null) { logger.error("Unable to find ES profile for participant with guid/altpid: {}, continuing w/ patch", patch.getParentId()); } + // List nameValues = processMultipleNameValues() + // declared nameValues above will be removed for (NameValue nameValue : patch.getNameValues()) { DBElement dbElement = PatchUtil.getColumnNameMap().get(nameValue.getName()); if (dbElement != null) { From 1fc1beed0e4829c56e1f58bceeac82d6a3688fcd Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 4 Oct 2021 17:12:08 +0400 Subject: [PATCH 007/405] refactor | continue separation of patchroute --- .../dsm/model/patch/BasePatch.java | 92 ++++++++++++++++++ .../dsm/model/patch/NullPatch.java | 6 +- .../dsm/model/patch/PatchFactory.java | 1 + .../dsm/model/patch/PrimaryKeyPatch.java | 97 ++++++++++++++++++- 4 files changed, 188 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index e3ab66616..58e3d680b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -1,30 +1,66 @@ package org.broadinstitute.dsm.model.patch; +import static org.broadinstitute.ddp.db.TransactionWrapper.inTransaction; + import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Optional; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; +import lombok.NonNull; +import org.apache.commons.lang3.StringUtils; +import org.broadinstitute.dsm.db.DDPInstance; +import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; +import org.broadinstitute.dsm.db.dao.queue.EventDao; +import org.broadinstitute.dsm.db.dao.settings.EventTypeDao; +import org.broadinstitute.dsm.db.dto.settings.EventTypeDto; import org.broadinstitute.dsm.db.structure.DBElement; +import org.broadinstitute.dsm.export.WorkflowForES; import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.Patch; +import org.broadinstitute.dsm.model.Value; +import org.broadinstitute.dsm.model.elasticsearch.ESProfile; +import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; +import org.broadinstitute.dsm.statics.ESObjectConstants; +import org.broadinstitute.dsm.util.ElasticSearchUtil; +import org.broadinstitute.dsm.util.EventUtil; import org.broadinstitute.dsm.util.NotificationUtil; +import org.broadinstitute.dsm.util.ParticipantUtil; import org.broadinstitute.dsm.util.PatchUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public abstract class BasePatch implements Patchable { + static final Logger logger = LoggerFactory.getLogger(BasePatch.class); + protected static final String PARTICIPANT_ID = "participantId"; protected static final String PRIMARY_KEY_ID = "primaryKeyId"; protected static final String NAME_VALUE = "NameValue"; protected static final String STATUS = "status"; + protected static final Gson GSON = new GsonBuilder().serializeNulls().create(); + protected Patch patch; + protected ESProfile profile; + protected DDPInstance ddpInstance; public BasePatch() { } public BasePatch(Patch patch) { this.patch = patch; + prepareNecessaryData(); + } + + private void prepareNecessaryData() { + ddpInstance = DDPInstance.getDDPInstance(patch.getRealm()); + profile = ElasticSearchUtil.getParticipantProfileByGuidOrAltPid(ddpInstance.getParticipantIndexES(), patch.getDdpParticipantId()) + .orElse(new ESProfile()); } abstract Optional processSingleNameValue(NameValue nameValue, DBElement dbElement); @@ -48,6 +84,62 @@ protected boolean hasQuestion(NameValue nameValue) { } + protected boolean hasProfileAndESWorkflowType(ESProfile profile, Value action) { + return ESObjectConstants.ELASTIC_EXPORT_WORKFLOWS.equals(action.getType()) && profile != null; + } + protected void writeESWorkflow(@NonNull Patch patch, @NonNull NameValue nameValue, @NonNull Value action, DDPInstance ddpInstance, + String esParticipantId) { + String status = nameValue.getValue() != null ? String.valueOf(nameValue.getValue()) : null; + if (StringUtils.isBlank(status)) { + return; + } + Map data = GSON.fromJson(status, new TypeToken>() { + }.getType()); + final ParticipantDataDao participantDataDao = new ParticipantDataDao(); + if (StringUtils.isNotBlank(action.getValue())) { + if (!patch.getFieldId().contains(FamilyMemberConstants.PARTICIPANTS)) { + ElasticSearchUtil.writeWorkflow(WorkflowForES.createInstance(ddpInstance, esParticipantId, action.getName(), action.getValue()), false); + } + else if (ParticipantUtil.matchesApplicantEmail(data.get(FamilyMemberConstants.COLLABORATOR_PARTICIPANT_ID), + participantDataDao.getParticipantDataByParticipantId(patch.getParentId()))) { + ElasticSearchUtil.writeWorkflow(WorkflowForES.createInstanceWithStudySpecificData(ddpInstance, + esParticipantId, action.getName(), data.get(action.getName()), new WorkflowForES.StudySpecificData( + data.get(FamilyMemberConstants.COLLABORATOR_PARTICIPANT_ID), + data.get(FamilyMemberConstants.FIRSTNAME), + data.get(FamilyMemberConstants.LASTNAME))), false); + } + } + else if (StringUtils.isNotBlank(action.getName()) && data.containsKey(action.getName())) { + if (!patch.getFieldId().contains(FamilyMemberConstants.PARTICIPANTS)) { + ElasticSearchUtil.writeWorkflow(WorkflowForES.createInstance(ddpInstance, esParticipantId, action.getName(), data.get(action.getName())), false); + } + else if (ParticipantUtil.matchesApplicantEmail(data.get(FamilyMemberConstants.COLLABORATOR_PARTICIPANT_ID), + participantDataDao.getParticipantDataByParticipantId(patch.getParentId()))) { + ElasticSearchUtil.writeWorkflow(WorkflowForES.createInstanceWithStudySpecificData(ddpInstance, + esParticipantId, action.getName(), data.get(action.getName()), new WorkflowForES.StudySpecificData( + data.get(FamilyMemberConstants.COLLABORATOR_PARTICIPANT_ID), + data.get(FamilyMemberConstants.FIRSTNAME), + data.get(FamilyMemberConstants.LASTNAME))), false); + } + } + } + protected void triggerParticipantEvent(DDPInstance ddpInstance, Patch patch, Value action){ + final EventDao eventDao = new EventDao(); + final EventTypeDao eventTypeDao = new EventTypeDao(); + Optional eventType = eventTypeDao.getEventTypeByEventTypeAndInstanceId(action.getName(), ddpInstance.getDdpInstanceId()); + eventType.ifPresent(eventTypeDto -> { + boolean participantHasTriggeredEventByEventType = eventDao.hasTriggeredEventByEventTypeAndDdpParticipantId(action.getName(), patch.getParentId()).orElse(false); + if (!participantHasTriggeredEventByEventType) { + inTransaction((conn) -> { + EventUtil.triggerDDP(conn, eventType, patch.getParentId()); + return null; + }); + } + else { + logger.info("Participant " + patch.getParentId() + " was already triggered for event type " + action.getName()); + } + }); + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java index d5033b4e3..d8eedfb63 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java @@ -9,12 +9,12 @@ public class NullPatch extends BasePatch { @Override - Optional processSingleNameValue(NameValue nameValue, DBElement dbElement) { + Optional processSingleNameValue(NameValue nameValue, DBElement dbElement) { return Optional.empty(); } @Override - Optional processMultipleNameValues(List nameValues) { - return Optional.empty(); + List processMultipleNameValues(List nameValues) { + return List.of(); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java index 5eb27967d..7e4e0e077 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java @@ -6,6 +6,7 @@ public class PatchFactory { public static Patchable makePatch(Patch patch) { Patchable patcher = new NullPatch(); + // switch cases here return patcher; } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java index 920f767aa..c0ede4e0a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java @@ -1,21 +1,36 @@ package org.broadinstitute.dsm.model.patch; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.CopyOnWriteArrayList; + +import org.broadinstitute.dsm.db.DDPInstance; +import org.broadinstitute.dsm.db.dao.settings.EventTypeDao; import org.broadinstitute.dsm.db.dao.user.UserDao; import org.broadinstitute.dsm.db.dto.user.UserDto; import org.broadinstitute.dsm.db.structure.DBElement; +import org.broadinstitute.dsm.export.WorkflowForES; import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.Patch; +import org.broadinstitute.dsm.model.Value; +import org.broadinstitute.dsm.model.elasticsearch.ESProfile; +import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; +import org.broadinstitute.dsm.model.settings.field.FieldSettings; +import org.broadinstitute.dsm.statics.ESObjectConstants; +import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.broadinstitute.dsm.util.NotificationUtil; import org.json.JSONArray; import org.json.JSONObject; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class PrimaryKeyPatch extends BasePatch { + private static final Logger logger = LoggerFactory.getLogger(PrimaryKeyPatch.class); + private NotificationUtil notificationUtil; public PrimaryKeyPatch(Patch patch, NotificationUtil notificationUtil) { @@ -69,6 +84,78 @@ private Optional sendNotificationEmailAndUpdateStatus(Patch patch, Na return maybeUpdatedNameValue; } + private void controlWorkflowByEmail(Patch patch, NameValue nameValue, DDPInstance ddpInstance, ESProfile profile) { + if (profile == null || nameValue.getValue() == null) { + return; + } + try { + Map pData = GSON.fromJson(nameValue.getValue().toString(), Map.class); + org.broadinstitute.dsm.model.participant.data.ParticipantData participantData = + new org.broadinstitute.dsm.model.participant.data.ParticipantData(Integer.parseInt(patch.getId()), + patch.getParentId(), Integer.parseInt(ddpInstance.getDdpInstanceId()), patch.getFieldId(), + pData); + + if (participantData.hasFamilyMemberApplicantEmail(profile)) { + writeFamilyMemberWorklow(patch, ddpInstance, profile, pData); + } else { + Map esMap = ElasticSearchUtil + .getObjectsMap(ddpInstance.getParticipantIndexES(), profile.getParticipantGuid(), + ESObjectConstants.WORKFLOWS); + if (Objects.isNull(esMap) || esMap.isEmpty()) return; + removeFamilyMemberWorkflowData(ddpInstance, profile, pData, esMap); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private void writeFamilyMemberWorklow(Patch patch, DDPInstance ddpInstance, ESProfile profile, Map pData) { + logger.info("Email in patch data matches participant profile email, will update workflows"); + int ddpInstanceIdByGuid = Integer.parseInt(ddpInstance.getDdpInstanceId()); + FieldSettings fieldSettings = new FieldSettings(); + pData.forEach((columnName, columnValue) -> { + if (!fieldSettings.isColumnExportable(ddpInstanceIdByGuid, columnName)) return; + if (!patch.getFieldId().contains(org.broadinstitute.dsm.model.participant.data.ParticipantData.FIELD_TYPE_PARTICIPANTS)) return; + // Use participant guid here to avoid multiple ES lookups. + ElasticSearchUtil.writeWorkflow(WorkflowForES.createInstanceWithStudySpecificData(ddpInstance, + profile.getParticipantGuid(), columnName, columnValue, new WorkflowForES.StudySpecificData( + pData.get(FamilyMemberConstants.COLLABORATOR_PARTICIPANT_ID), + pData.get(FamilyMemberConstants.FIRSTNAME), + pData.get(FamilyMemberConstants.LASTNAME))), false); + }); + } + + private void removeFamilyMemberWorkflowData(DDPInstance ddpInstance, ESProfile profile, Map pData, Map esMap) throws + IOException { + logger.info("Email in patch data does not match participant profile email, will remove workflows"); + CopyOnWriteArrayList> workflowsList = new CopyOnWriteArrayList<>((List>) esMap.get(ESObjectConstants.WORKFLOWS)); + int startingSize = workflowsList.size(); + workflowsList.forEach(workflow -> { + Map workflowDataMap = (Map) workflow.get(ESObjectConstants.DATA); + String collaboratorParticipantId = workflowDataMap.get(ESObjectConstants.SUBJECT_ID); + if (Objects.isNull(collaboratorParticipantId)) return; + if (collaboratorParticipantId.equals(pData.get(FamilyMemberConstants.COLLABORATOR_PARTICIPANT_ID))) { + workflowsList.remove(workflow); + } + }); + if (startingSize != workflowsList.size()) { + esMap.put(ESObjectConstants.WORKFLOWS, workflowsList); + // Use participant guid here to avoid another ES lookup. + ElasticSearchUtil.updateRequest(profile.getParticipantGuid(), ddpInstance.getParticipantIndexES(), esMap); + } + } + + private void writeESWorkflowElseTriggerParticipantEvent(Patch patch, DDPInstance ddpInstance, ESProfile profile, NameValue nameValue) { + for (Value action : patch.getActions()) { + if (hasProfileAndESWorkflowType(profile, action)) { + writeESWorkflow(patch, nameValue, action, ddpInstance, profile.getParticipantGuid()); + } + else if (EventTypeDao.EVENT.equals(action.getType())) { + triggerParticipantEvent(ddpInstance, patch, action); + } + } + } + private boolean isSent(JSONObject question) { return question.optString(STATUS) != null && question.optString(STATUS).equals("sent"); } From 295c968e771b68ecca7ea4c8463bcba3f7cfb143 Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 4 Oct 2021 18:50:07 +0400 Subject: [PATCH 008/405] refactor | refine BasePatch.java by adding new abstract/base methods --- .../dsm/model/patch/BasePatch.java | 33 ++++++++-- .../dsm/model/patch/NullPatch.java | 7 +- .../dsm/model/patch/PatchFactory.java | 10 ++- .../dsm/model/patch/PrimaryKeyPatch.java | 65 +++++++++++++++++-- .../broadinstitute/dsm/route/PatchRoute.java | 6 ++ 5 files changed, 106 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index 58e3d680b..5d9cd6741 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -12,6 +12,7 @@ import com.google.gson.reflect.TypeToken; import lombok.NonNull; import org.apache.commons.lang3.StringUtils; +import org.broadinstitute.ddp.handlers.util.Result; import org.broadinstitute.dsm.db.DDPInstance; import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; import org.broadinstitute.dsm.db.dao.queue.EventDao; @@ -27,14 +28,13 @@ import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.broadinstitute.dsm.util.EventUtil; -import org.broadinstitute.dsm.util.NotificationUtil; import org.broadinstitute.dsm.util.ParticipantUtil; import org.broadinstitute.dsm.util.PatchUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public abstract class BasePatch implements Patchable { +public abstract class BasePatch { static final Logger logger = LoggerFactory.getLogger(BasePatch.class); @@ -44,7 +44,6 @@ public abstract class BasePatch implements Patchable { protected static final String STATUS = "status"; protected static final Gson GSON = new GsonBuilder().serializeNulls().create(); - protected Patch patch; protected ESProfile profile; protected DDPInstance ddpInstance; @@ -57,20 +56,36 @@ public BasePatch(Patch patch) { prepareNecessaryData(); } + public abstract Object doPatch(); + + abstract Optional processEachNameValue(NameValue nameValue, DBElement dbElement); + + abstract Object processEachNameValue(); + private void prepareNecessaryData() { ddpInstance = DDPInstance.getDDPInstance(patch.getRealm()); profile = ElasticSearchUtil.getParticipantProfileByGuidOrAltPid(ddpInstance.getParticipantIndexES(), patch.getDdpParticipantId()) - .orElse(new ESProfile()); + .orElse(null); } - abstract Optional processSingleNameValue(NameValue nameValue, DBElement dbElement); + Optional handleSingleNameValue() { + Optional object; + DBElement dbElement = PatchUtil.getColumnNameMap().get(patch.getNameValue().getName()); + if (dbElement != null) { + object = Optional.of(processEachNameValue()); + } + else { + throw new RuntimeException("DBElement not found in ColumnNameMap: " + patch.getNameValue().getName()); + } + return object; + } List processMultipleNameValues(List nameValues) { List updatedNameValues = new ArrayList<>(); for (NameValue nameValue : patch.getNameValues()) { DBElement dbElement = PatchUtil.getColumnNameMap().get(nameValue.getName()); if (dbElement != null) { - processSingleNameValue(nameValue, dbElement).ifPresent(updatedNameValues::add); + processEachNameValue(nameValue, dbElement).ifPresent(updatedNameValues::add); } else { throw new RuntimeException("DBElement not found in ColumnNameMap: " + nameValue.getName()); @@ -142,4 +157,10 @@ protected void triggerParticipantEvent(DDPInstance ddpInstance, Patch patch, Val } }); } + + protected boolean isNameValuePairs() { + //TODO -> could be changed later after clarification + return patch.getNameValues() != null && !patch.getNameValues().isEmpty(); + } + } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java index d8eedfb63..d53fcaeb4 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java @@ -9,7 +9,12 @@ public class NullPatch extends BasePatch { @Override - Optional processSingleNameValue(NameValue nameValue, DBElement dbElement) { + public Object doPatch() { + return new Object(); + } + + @Override + Optional processEachNameValue(NameValue nameValue, DBElement dbElement) { return Optional.empty(); } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java index 7e4e0e077..ba8b1ff0c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java @@ -4,13 +4,17 @@ public class PatchFactory { - public static Patchable makePatch(Patch patch) { - Patchable patcher = new NullPatch(); - + public static BasePatch makePatch(Patch patch) { + BasePatch patcher = new NullPatch(); + + + // switch cases here return patcher; } + + } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java index c0ede4e0a..36487e5cb 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java @@ -1,12 +1,10 @@ package org.broadinstitute.dsm.model.patch; import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; +import java.util.*; import java.util.concurrent.CopyOnWriteArrayList; +import org.broadinstitute.ddp.handlers.util.Result; import org.broadinstitute.dsm.db.DDPInstance; import org.broadinstitute.dsm.db.dao.settings.EventTypeDao; import org.broadinstitute.dsm.db.dao.user.UserDao; @@ -22,6 +20,7 @@ import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.broadinstitute.dsm.util.NotificationUtil; +import org.broadinstitute.dsm.util.PatchUtil; import org.json.JSONArray; import org.json.JSONObject; import org.slf4j.Logger; @@ -39,7 +38,58 @@ public PrimaryKeyPatch(Patch patch, NotificationUtil notificationUtil) { } @Override - Optional processSingleNameValue(NameValue nameValue, DBElement dbElement) { + public Object doPatch() { + if (isNameValuePairs()) { + List nameValues = new ArrayList<>(); + ESProfile profile = ElasticSearchUtil.getParticipantProfileByGuidOrAltPid(ddpInstance.getParticipantIndexES(), patch.getDdpParticipantId()) + .orElse(null); + if (profile == null) { + logger.error("Unable to find ES profile for participant with guid/altpid: {}, continuing w/ patch", patch.getParentId()); + } + // List nameValues = processMultipleNameValues() + // declared nameValues above will be removed + for (NameValue nameValue : patch.getNameValues()) { + DBElement dbElement = PatchUtil.getColumnNameMap().get(nameValue.getName()); + if (dbElement != null) { + // basePatch.processMultipleNameValues(patch.getNameValues()) + if (!Patch.patch(patch.getId(), patch.getUser(), nameValue, dbElement)) { + throw new RuntimeException("An error occurred while attempting to patch "); + } + if (hasQuestion(nameValue)) { + sendNotificationEmailAndUpdateStatus(patch, nameValue, dbElement); + } + controlWorkflowByEmail(patch, nameValue, ddpInstance, profile); + if (patch.getActions() != null) { + writeESWorkflowElseTriggerParticipantEvent(patch, ddpInstance, profile, nameValue); + } + } + else { + throw new RuntimeException("DBElement not found in ColumnNameMap: " + nameValue.getName()); + } + } + return new Result(200, GSON.toJson(nameValues)); + } + else { + // mr changes + DBElement dbElement = PatchUtil.getColumnNameMap().get(patch.getNameValue().getName()); + if (dbElement != null) { + if (Patch.patch(patch.getId(), patch.getUser(), patch.getNameValue(), dbElement)) { + List nameValues = setWorkflowRelatedFields(patch); + writeDSMRecordsToES(patch, ddpInstance); + //return nameValues with nulls + return new Result(200, gson.toJson(nameValues)); + } + } + else { + throw new RuntimeException("DBElement not found in ColumnNameMap: " + patch.getNameValue().getName()); + } + } + + return null; + } + + @Override + Optional processEachNameValue(NameValue nameValue, DBElement dbElement) { Optional maybeUpdatedNameValue = Optional.empty(); if (!Patch.patch(patch.getId(), patch.getUser(), nameValue, dbElement)) { throw new RuntimeException("An error occurred while attempting to patch "); @@ -55,6 +105,11 @@ Optional processSingleNameValue(NameValue nameValue, DBElement dbElem return maybeUpdatedNameValue; } + @Override + Optional handleSingleNameValue() { + return null; + } + private Optional sendNotificationEmailAndUpdateStatus(Patch patch, NameValue nameValue, DBElement dbElement) { Optional maybeUpdatedNameValue = Optional.empty(); UserDto userDto = new UserDao().getUserByEmail(patch.getUser()).orElseThrow(); diff --git a/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java b/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java index 7f7cef71b..d5f934149 100644 --- a/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java @@ -26,6 +26,9 @@ import org.broadinstitute.dsm.model.Value; import org.broadinstitute.dsm.model.elasticsearch.ESProfile; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; +import org.broadinstitute.dsm.model.patch.BasePatch; +import org.broadinstitute.dsm.model.patch.PatchFactory; +import org.broadinstitute.dsm.model.patch.Patchable; import org.broadinstitute.dsm.model.settings.field.FieldSettings; import org.broadinstitute.dsm.security.RequestHandler; import org.broadinstitute.dsm.statics.DBConstants; @@ -77,6 +80,9 @@ public Object processRequest(Request request, Response response, String userId) try { String requestBody = request.body(); Patch patch = gson.fromJson(requestBody, Patch.class); + BasePatch patcher = PatchFactory.makePatch(patch); + patcher.doPatch(); + if (hasPrimaryKey(patch)) { //multiple values are changing DDPInstance ddpInstance = DDPInstance.getDDPInstance(patch.getRealm()); From dc3887c3d7384f7fc03bf2f2a4746654dd44be7a Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 4 Oct 2021 19:18:07 +0400 Subject: [PATCH 009/405] refactor | continue moving general parts into base patch --- .../dsm/model/patch/BasePatch.java | 93 +++++++++++++++++-- .../dsm/model/patch/PrimaryKeyPatch.java | 78 +++++++++++----- .../broadinstitute/dsm/util/PatchUtil.java | 7 +- 3 files changed, 145 insertions(+), 33 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index 5d9cd6741..249817fb8 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -12,8 +12,8 @@ import com.google.gson.reflect.TypeToken; import lombok.NonNull; import org.apache.commons.lang3.StringUtils; -import org.broadinstitute.ddp.handlers.util.Result; import org.broadinstitute.dsm.db.DDPInstance; +import org.broadinstitute.dsm.db.OncHistoryDetail; import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; import org.broadinstitute.dsm.db.dao.queue.EventDao; import org.broadinstitute.dsm.db.dao.settings.EventTypeDao; @@ -60,7 +60,7 @@ public BasePatch(Patch patch) { abstract Optional processEachNameValue(NameValue nameValue, DBElement dbElement); - abstract Object processEachNameValue(); + abstract Object handleSingleNameValue(DBElement dbElement); private void prepareNecessaryData() { ddpInstance = DDPInstance.getDDPInstance(patch.getRealm()); @@ -68,16 +68,16 @@ private void prepareNecessaryData() { .orElse(null); } - Optional handleSingleNameValue() { - Optional object; + Optional processSingleNameValue() { + Optional result; DBElement dbElement = PatchUtil.getColumnNameMap().get(patch.getNameValue().getName()); if (dbElement != null) { - object = Optional.of(processEachNameValue()); + result = Optional.of(handleSingleNameValue(dbElement)); } else { throw new RuntimeException("DBElement not found in ColumnNameMap: " + patch.getNameValue().getName()); } - return object; + return result; } List processMultipleNameValues(List nameValues) { @@ -163,4 +163,85 @@ protected boolean isNameValuePairs() { return patch.getNameValues() != null && !patch.getNameValues().isEmpty(); } + protected List setWorkflowRelatedFields(@NonNull Patch patch) { + List nameValues = new ArrayList<>(); + //mr request workflow + if (patch.getNameValue().getName().equals("m.faxSent")) { + nameValues.add(setAdditionalValue("m.faxSentBy", patch, patch.getUser())); + nameValues.add(setAdditionalValue("m.faxConfirmed", patch, patch.getNameValue().getValue())); + } + else if (patch.getNameValue().getName().equals("m.faxSent2")) { + nameValues.add(setAdditionalValue("m.faxSent2By", patch, patch.getUser())); + nameValues.add(setAdditionalValue("m.faxConfirmed2", patch, patch.getNameValue().getValue())); + } + else if (patch.getNameValue().getName().equals("m.faxSent3")) { + nameValues.add(setAdditionalValue("m.faxSent3By", patch, patch.getUser())); + nameValues.add(setAdditionalValue("m.faxConfirmed3", patch, patch.getNameValue().getValue())); + } + //tissue request workflow + else if (patch.getNameValue().getName().equals("oD.tFaxSent")) { + nameValues.add(setAdditionalValue("oD.tFaxSentBy", patch, patch.getUser())); + nameValues.add(setAdditionalValue("oD.tFaxConfirmed", patch, patch.getNameValue().getValue())); + nameValues.add(setAdditionalValue("oD.request", patch, "sent")); + } + else if (patch.getNameValue().getName().equals("oD.tFaxSent2")) { + nameValues.add(setAdditionalValue("oD.tFaxSent2By", patch, patch.getUser())); + nameValues.add(setAdditionalValue("oD.tFaxConfirmed2", patch, patch.getNameValue().getValue())); + nameValues.add(setAdditionalValue("oD.request", patch, "sent")); + } + else if (patch.getNameValue().getName().equals("oD.tFaxSent3")) { + nameValues.add(setAdditionalValue("oD.tFaxSent3By", patch, patch.getUser())); + nameValues.add(setAdditionalValue("oD.tFaxConfirmed3", patch, patch.getNameValue().getValue())); + nameValues.add(setAdditionalValue("oD.request", patch, "sent")); + } + else if (patch.getNameValue().getName().equals("oD.tissueReceived")) { + nameValues.add(setAdditionalValue("oD.request", patch, "received")); + } + else if (patch.getNameValue().getName().equals("t.tissueReturnDate")) { + if (StringUtils.isNotBlank(patch.getNameValue().getValue().toString())) { + nameValues.add(setAdditionalValue("oD.request", new Patch(patch.getParentId(), PARTICIPANT_ID, + null, patch.getUser(), patch.getNameValue(), patch.getNameValues(), patch.getDdpParticipantId()), "returned")); + } + else { + Boolean hasReceivedDate = OncHistoryDetail.hasReceivedDate(patch); + + if (hasReceivedDate) { + nameValues.add(setAdditionalValue("oD.request", new Patch(patch.getParentId(), PARTICIPANT_ID, + null, patch.getUser(), patch.getNameValue(), patch.getNameValues(), patch.getDdpParticipantId()), "received")); + } + else { + nameValues.add(setAdditionalValue("oD.request", new Patch(patch.getParentId(), PARTICIPANT_ID, + null, patch.getUser(), patch.getNameValue(), patch.getNameValues(), patch.getDdpParticipantId()), "sent")); + } + } + } + else if (patch.getNameValue().getName().equals("oD.unableToObtain") && (boolean) patch.getNameValue().getValue()) { + } + else if (patch.getNameValue().getName().equals("oD.unableToObtain") && !(boolean) patch.getNameValue().getValue()) { + Boolean hasReceivedDate = OncHistoryDetail.hasReceivedDate(patch); + + if (hasReceivedDate) { + nameValues.add(setAdditionalValue("oD.request", new Patch(patch.getId(), PARTICIPANT_ID, + patch.getParentId(), patch.getUser(), patch.getNameValue(), patch.getNameValues(), patch.getDdpParticipantId()), "received")); + } + else { + nameValues.add(setAdditionalValue("oD.request", new Patch(patch.getId(), PARTICIPANT_ID, + patch.getParentId(), patch.getUser(), patch.getNameValue(), patch.getNameValues(), patch.getDdpParticipantId()), "sent")); + } + } + return nameValues; + } + + private NameValue setAdditionalValue(String additionalValue, @NonNull Patch patch, @NonNull Object value) { + DBElement dbElement = PatchUtil.getColumnNameMap().get(additionalValue); + if (dbElement != null) { + NameValue nameValue = new NameValue(additionalValue, value); + Patch.patch(patch.getId(), patch.getUser(), nameValue, dbElement); + return nameValue; + } + else { + throw new RuntimeException("DBElement not found in ColumnNameMap: " + additionalValue); + } + } + } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java index 36487e5cb..66dafbc0a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java @@ -4,6 +4,7 @@ import java.util.*; import java.util.concurrent.CopyOnWriteArrayList; +import lombok.NonNull; import org.broadinstitute.ddp.handlers.util.Result; import org.broadinstitute.dsm.db.DDPInstance; import org.broadinstitute.dsm.db.dao.settings.EventTypeDao; @@ -17,6 +18,7 @@ import org.broadinstitute.dsm.model.elasticsearch.ESProfile; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import org.broadinstitute.dsm.model.settings.field.FieldSettings; +import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.broadinstitute.dsm.util.NotificationUtil; @@ -40,36 +42,17 @@ public PrimaryKeyPatch(Patch patch, NotificationUtil notificationUtil) { @Override public Object doPatch() { if (isNameValuePairs()) { - List nameValues = new ArrayList<>(); ESProfile profile = ElasticSearchUtil.getParticipantProfileByGuidOrAltPid(ddpInstance.getParticipantIndexES(), patch.getDdpParticipantId()) .orElse(null); if (profile == null) { logger.error("Unable to find ES profile for participant with guid/altpid: {}, continuing w/ patch", patch.getParentId()); } - // List nameValues = processMultipleNameValues() - // declared nameValues above will be removed - for (NameValue nameValue : patch.getNameValues()) { - DBElement dbElement = PatchUtil.getColumnNameMap().get(nameValue.getName()); - if (dbElement != null) { - // basePatch.processMultipleNameValues(patch.getNameValues()) - if (!Patch.patch(patch.getId(), patch.getUser(), nameValue, dbElement)) { - throw new RuntimeException("An error occurred while attempting to patch "); - } - if (hasQuestion(nameValue)) { - sendNotificationEmailAndUpdateStatus(patch, nameValue, dbElement); - } - controlWorkflowByEmail(patch, nameValue, ddpInstance, profile); - if (patch.getActions() != null) { - writeESWorkflowElseTriggerParticipantEvent(patch, ddpInstance, profile, nameValue); - } - } - else { - throw new RuntimeException("DBElement not found in ColumnNameMap: " + nameValue.getName()); - } - } - return new Result(200, GSON.toJson(nameValues)); + List updatedNameValues = processMultipleNameValues(patch.getNameValues()); + return new Result(200, GSON.toJson(updatedNameValues)); } else { + Optional maybeNameValue = processSingleNameValue(); + return maybeNameValue.orElse(); // mr changes DBElement dbElement = PatchUtil.getColumnNameMap().get(patch.getNameValue().getName()); if (dbElement != null) { @@ -106,8 +89,53 @@ Optional processEachNameValue(NameValue nameValue, DBElement dbElemen } @Override - Optional handleSingleNameValue() { - return null; + Object handleSingleNameValue(DBElement dbElement) { + List nameValues = new ArrayList<>(); + if (Patch.patch(patch.getId(), patch.getUser(), patch.getNameValue(), dbElement)) { + nameValues.addAll(setWorkflowRelatedFields(patch)); + writeDSMRecordsToES(); + //return nameValues with nulls + return nameValues; + } + return nameValues; + } + + private void writeDSMRecordsToES() { + NameValue nameValue = patch.getNameValue(); + String name = nameValue.getName().substring(nameValue.getName().lastIndexOf('.') + 1); + String type = getTypeFrom(nameValue); + if (type == null) return; + String value = nameValue.getValue().toString(); + Map nameValueMap = new HashMap<>(); + nameValueMap.put(name, value); + if (isMedicalRecord(name, type)) { + ElasticSearchUtil.writeDsmRecord(ddpInstance, Integer.parseInt(patch.getId()), patch.getDdpParticipantId(), + ESObjectConstants.MEDICAL_RECORDS, ESObjectConstants.MEDICAL_RECORDS_ID, nameValueMap); + } + else if (DBConstants.DDP_ONC_HISTORY_DETAIL_ALIAS.equals(type)) { + // TODO relationship between DDP_ONC_HISTORY_DETAIL_ALIAS and TISSUE_RECORDS_FIELD_NAMES + if (ESObjectConstants.TISSUE_RECORDS_FIELD_NAMES.contains(name)) { + if (PARTICIPANT_ID.equals(patch.getParent())) { + ElasticSearchUtil.writeDsmRecord(ddpInstance, Integer.parseInt(patch.getId()), patch.getDdpParticipantId(), + ESObjectConstants.TISSUE_RECORDS, ESObjectConstants.TISSUE_RECORDS_ID, nameValueMap); + } + } + } + } + + private String getTypeFrom(NameValue nameValue) { + String type = null; + if (nameValue.getName().indexOf('.') != -1) { + type = nameValue.getName().substring(0, nameValue.getName().indexOf('.')); + } + else { + return null; + } + return type; + } + + private boolean isMedicalRecord(String name, String type) { + return DBConstants.DDP_MEDICAL_RECORD_ALIAS.equals(type) && ESObjectConstants.MEDICAL_RECORDS_FIELD_NAMES.contains(name); } private Optional sendNotificationEmailAndUpdateStatus(Patch patch, NameValue nameValue, DBElement dbElement) { diff --git a/src/main/java/org/broadinstitute/dsm/util/PatchUtil.java b/src/main/java/org/broadinstitute/dsm/util/PatchUtil.java index 6ffd4f0fd..08cd43afa 100644 --- a/src/main/java/org/broadinstitute/dsm/util/PatchUtil.java +++ b/src/main/java/org/broadinstitute/dsm/util/PatchUtil.java @@ -25,7 +25,7 @@ public class PatchUtil { private static Map dataBaseMap; private static Set tableAliases = new HashSet<>(); - public PatchUtil() { + static { columnNameMap = new HashMap<>(); dataBaseMap = new HashMap<>(); getColumnNames(Participant.class); @@ -43,6 +43,9 @@ public PatchUtil() { getColumnNames(ParticipantData.class); logger.info("Loaded patch utils"); } + + public PatchUtil() { + } public static Map getColumnNameMap() { if (columnNameMap != null && !columnNameMap.isEmpty()) { @@ -65,7 +68,7 @@ public static Map getDataBaseMap() { return null; } - private void getColumnNames(Class obj) { + private static void getColumnNames(Class obj) { String tableName = null; String tableAlias = null; String primaryKey = null; From f0747a999825ecfd06e957e33ce13768ad365858 Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 5 Oct 2021 15:16:12 +0400 Subject: [PATCH 010/405] refactor | create AbstractionPatch for handling the update for abstraction related stuff --- .../dsm/model/patch/AbstractionPatch.java | 35 +++++ .../dsm/model/patch/BasePatch.java | 18 +-- .../dsm/model/patch/NullPatch.java | 5 + .../dsm/model/patch/PatchFactory.java | 14 +- .../dsm/model/patch/PrimaryKeyPatch.java | 126 ++++++++---------- .../broadinstitute/dsm/route/PatchRoute.java | 119 +++++++++-------- 6 files changed, 179 insertions(+), 138 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java new file mode 100644 index 000000000..e55b9b01f --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java @@ -0,0 +1,35 @@ +package org.broadinstitute.dsm.model.patch; + +import org.broadinstitute.dsm.db.structure.DBElement; +import org.broadinstitute.dsm.model.AbstractionWrapper; +import org.broadinstitute.dsm.model.NameValue; +import org.broadinstitute.dsm.model.Patch; +import spark.utils.StringUtils; + +import java.util.Optional; + +public class AbstractionPatch extends BasePatch { + + String primaryKeyId; + + @Override + public Object doPatch() { + return null; + } + + @Override + Object handleSingleNameValue(DBElement dbElement) { + return null; + } + + @Override + Optional processEachNameValue(NameValue nameValue, DBElement dbElement) { + if (StringUtils.isBlank(primaryKeyId)) { + primaryKeyId = AbstractionWrapper.createNewAbstractionFieldValue(patch.getParentId(), patch.getFieldId(), patch.getUser(), nameValue, dbElement); + } + if (!Patch.patch(primaryKeyId, patch.getUser(), nameValue, dbElement)) { + throw new RuntimeException("An error occurred while attempting to patch "); + } + return Optional.empty(); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index 249817fb8..ce74fe4ec 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory; -public abstract class BasePatch { +public abstract class BasePatch { static final Logger logger = LoggerFactory.getLogger(BasePatch.class); @@ -56,11 +56,10 @@ public BasePatch(Patch patch) { prepareNecessaryData(); } - public abstract Object doPatch(); + public abstract T doPatch(); - abstract Optional processEachNameValue(NameValue nameValue, DBElement dbElement); - abstract Object handleSingleNameValue(DBElement dbElement); + abstract T handleSingleNameValue(DBElement dbElement); private void prepareNecessaryData() { ddpInstance = DDPInstance.getDDPInstance(patch.getRealm()); @@ -68,8 +67,8 @@ private void prepareNecessaryData() { .orElse(null); } - Optional processSingleNameValue() { - Optional result; + Optional processSingleNameValue() { + Optional result; DBElement dbElement = PatchUtil.getColumnNameMap().get(patch.getNameValue().getName()); if (dbElement != null) { result = Optional.of(handleSingleNameValue(dbElement)); @@ -80,8 +79,8 @@ Optional processSingleNameValue() { return result; } - List processMultipleNameValues(List nameValues) { - List updatedNameValues = new ArrayList<>(); + T processMultipleNameValues() { + List updatedNameValues = new ArrayList<>(); for (NameValue nameValue : patch.getNameValues()) { DBElement dbElement = PatchUtil.getColumnNameMap().get(nameValue.getName()); if (dbElement != null) { @@ -94,11 +93,12 @@ List processMultipleNameValues(List nameValues) { return updatedNameValues; } + abstract Optional processEachNameValue(NameValue nameValue, DBElement dbElement); + protected boolean hasQuestion(NameValue nameValue) { return nameValue.getName().contains("question"); } - protected boolean hasProfileAndESWorkflowType(ESProfile profile, Value action) { return ESObjectConstants.ELASTIC_EXPORT_WORKFLOWS.equals(action.getType()) && profile != null; } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java index d53fcaeb4..d98af1958 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java @@ -18,6 +18,11 @@ Optional processEachNameValue(NameValue nameValue, DBElement dbElemen return Optional.empty(); } + @Override + Object handleSingleNameValue(DBElement dbElement) { + return new Object(); + } + @Override List processMultipleNameValues(List nameValues) { return List.of(); diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java index ba8b1ff0c..24022048f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java @@ -1,18 +1,24 @@ package org.broadinstitute.dsm.model.patch; +import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.Patch; +import org.broadinstitute.dsm.util.NotificationUtil; public class PatchFactory { - public static BasePatch makePatch(Patch patch) { + public static BasePatch makePatch(Patch patch, NotificationUtil notificationUtil) { BasePatch patcher = new NullPatch(); - - - + if (hasPrimaryKey(patch)) { + patcher = new PrimaryKeyPatch(patch, notificationUtil); + } // switch cases here return patcher; } + private static boolean hasPrimaryKey(Patch patch) { + return StringUtils.isNotBlank(patch.getId()); + } + diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java index 66dafbc0a..8a9368daf 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java @@ -4,7 +4,6 @@ import java.util.*; import java.util.concurrent.CopyOnWriteArrayList; -import lombok.NonNull; import org.broadinstitute.ddp.handlers.util.Result; import org.broadinstitute.dsm.db.DDPInstance; import org.broadinstitute.dsm.db.dao.settings.EventTypeDao; @@ -22,7 +21,6 @@ import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.broadinstitute.dsm.util.NotificationUtil; -import org.broadinstitute.dsm.util.PatchUtil; import org.json.JSONArray; import org.json.JSONObject; import org.slf4j.Logger; @@ -32,7 +30,7 @@ public class PrimaryKeyPatch extends BasePatch { private static final Logger logger = LoggerFactory.getLogger(PrimaryKeyPatch.class); - private NotificationUtil notificationUtil; + private final NotificationUtil notificationUtil; public PrimaryKeyPatch(Patch patch, NotificationUtil notificationUtil) { super(patch); @@ -52,23 +50,8 @@ public Object doPatch() { } else { Optional maybeNameValue = processSingleNameValue(); - return maybeNameValue.orElse(); - // mr changes - DBElement dbElement = PatchUtil.getColumnNameMap().get(patch.getNameValue().getName()); - if (dbElement != null) { - if (Patch.patch(patch.getId(), patch.getUser(), patch.getNameValue(), dbElement)) { - List nameValues = setWorkflowRelatedFields(patch); - writeDSMRecordsToES(patch, ddpInstance); - //return nameValues with nulls - return new Result(200, gson.toJson(nameValues)); - } - } - else { - throw new RuntimeException("DBElement not found in ColumnNameMap: " + patch.getNameValue().getName()); - } + return maybeNameValue.orElse(null); } - - return null; } @Override @@ -84,60 +67,9 @@ Optional processEachNameValue(NameValue nameValue, DBElement dbElemen if (patch.getActions() != null) { writeESWorkflowElseTriggerParticipantEvent(patch, ddpInstance, profile, nameValue); } - return maybeUpdatedNameValue; } - @Override - Object handleSingleNameValue(DBElement dbElement) { - List nameValues = new ArrayList<>(); - if (Patch.patch(patch.getId(), patch.getUser(), patch.getNameValue(), dbElement)) { - nameValues.addAll(setWorkflowRelatedFields(patch)); - writeDSMRecordsToES(); - //return nameValues with nulls - return nameValues; - } - return nameValues; - } - - private void writeDSMRecordsToES() { - NameValue nameValue = patch.getNameValue(); - String name = nameValue.getName().substring(nameValue.getName().lastIndexOf('.') + 1); - String type = getTypeFrom(nameValue); - if (type == null) return; - String value = nameValue.getValue().toString(); - Map nameValueMap = new HashMap<>(); - nameValueMap.put(name, value); - if (isMedicalRecord(name, type)) { - ElasticSearchUtil.writeDsmRecord(ddpInstance, Integer.parseInt(patch.getId()), patch.getDdpParticipantId(), - ESObjectConstants.MEDICAL_RECORDS, ESObjectConstants.MEDICAL_RECORDS_ID, nameValueMap); - } - else if (DBConstants.DDP_ONC_HISTORY_DETAIL_ALIAS.equals(type)) { - // TODO relationship between DDP_ONC_HISTORY_DETAIL_ALIAS and TISSUE_RECORDS_FIELD_NAMES - if (ESObjectConstants.TISSUE_RECORDS_FIELD_NAMES.contains(name)) { - if (PARTICIPANT_ID.equals(patch.getParent())) { - ElasticSearchUtil.writeDsmRecord(ddpInstance, Integer.parseInt(patch.getId()), patch.getDdpParticipantId(), - ESObjectConstants.TISSUE_RECORDS, ESObjectConstants.TISSUE_RECORDS_ID, nameValueMap); - } - } - } - } - - private String getTypeFrom(NameValue nameValue) { - String type = null; - if (nameValue.getName().indexOf('.') != -1) { - type = nameValue.getName().substring(0, nameValue.getName().indexOf('.')); - } - else { - return null; - } - return type; - } - - private boolean isMedicalRecord(String name, String type) { - return DBConstants.DDP_MEDICAL_RECORD_ALIAS.equals(type) && ESObjectConstants.MEDICAL_RECORDS_FIELD_NAMES.contains(name); - } - private Optional sendNotificationEmailAndUpdateStatus(Patch patch, NameValue nameValue, DBElement dbElement) { Optional maybeUpdatedNameValue = Optional.empty(); UserDto userDto = new UserDao().getUserByEmail(patch.getUser()).orElseThrow(); @@ -167,6 +99,10 @@ private Optional sendNotificationEmailAndUpdateStatus(Patch patch, Na return maybeUpdatedNameValue; } + private boolean isSent(JSONObject question) { + return question.optString(STATUS) != null && question.optString(STATUS).equals("sent"); + } + private void controlWorkflowByEmail(Patch patch, NameValue nameValue, DDPInstance ddpInstance, ESProfile profile) { if (profile == null || nameValue.getValue() == null) { return; @@ -239,8 +175,54 @@ else if (EventTypeDao.EVENT.equals(action.getType())) { } } - private boolean isSent(JSONObject question) { - return question.optString(STATUS) != null && question.optString(STATUS).equals("sent"); + @Override + Object handleSingleNameValue(DBElement dbElement) { + List nameValues = new ArrayList<>(); + if (Patch.patch(patch.getId(), patch.getUser(), patch.getNameValue(), dbElement)) { + nameValues.addAll(setWorkflowRelatedFields(patch)); + writeDSMRecordsToES(); + //return nameValues with nulls + return nameValues; + } + return nameValues; + } + + private void writeDSMRecordsToES() { + NameValue nameValue = patch.getNameValue(); + String name = nameValue.getName().substring(nameValue.getName().lastIndexOf('.') + 1); + String type = getTypeFrom(nameValue); + if (type == null) return; + String value = nameValue.getValue().toString(); + Map nameValueMap = new HashMap<>(); + nameValueMap.put(name, value); + if (isMedicalRecord(name, type)) { + ElasticSearchUtil.writeDsmRecord(ddpInstance, Integer.parseInt(patch.getId()), patch.getDdpParticipantId(), + ESObjectConstants.MEDICAL_RECORDS, ESObjectConstants.MEDICAL_RECORDS_ID, nameValueMap); + } + else if (DBConstants.DDP_ONC_HISTORY_DETAIL_ALIAS.equals(type)) { + // TODO relationship between DDP_ONC_HISTORY_DETAIL_ALIAS and TISSUE_RECORDS_FIELD_NAMES + if (ESObjectConstants.TISSUE_RECORDS_FIELD_NAMES.contains(name)) { + if (PARTICIPANT_ID.equals(patch.getParent())) { + ElasticSearchUtil.writeDsmRecord(ddpInstance, Integer.parseInt(patch.getId()), patch.getDdpParticipantId(), + ESObjectConstants.TISSUE_RECORDS, ESObjectConstants.TISSUE_RECORDS_ID, nameValueMap); + } + } + } + } + + private String getTypeFrom(NameValue nameValue) { + String type = null; + if (nameValue.getName().indexOf('.') != -1) { + type = nameValue.getName().substring(0, nameValue.getName().indexOf('.')); + } + else { + return null; + } + return type; + } + + private boolean isMedicalRecord(String name, String type) { + return DBConstants.DDP_MEDICAL_RECORD_ALIAS.equals(type) && ESObjectConstants.MEDICAL_RECORDS_FIELD_NAMES.contains(name); } } diff --git a/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java b/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java index d5f934149..b73f5c7dd 100644 --- a/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java @@ -80,61 +80,61 @@ public Object processRequest(Request request, Response response, String userId) try { String requestBody = request.body(); Patch patch = gson.fromJson(requestBody, Patch.class); - BasePatch patcher = PatchFactory.makePatch(patch); + BasePatch patcher = PatchFactory.makePatch(patch, notificationUtil); patcher.doPatch(); - if (hasPrimaryKey(patch)) { - //multiple values are changing - DDPInstance ddpInstance = DDPInstance.getDDPInstance(patch.getRealm()); - if (isNameValuePairs(patch)) { - List nameValues = new ArrayList<>(); - ESProfile profile = ElasticSearchUtil.getParticipantProfileByGuidOrAltPid(ddpInstance.getParticipantIndexES(), patch.getDdpParticipantId()) - .orElse(null); - if (profile == null) { - logger.error("Unable to find ES profile for participant with guid/altpid: {}, continuing w/ patch", patch.getParentId()); - } - // List nameValues = processMultipleNameValues() - // declared nameValues above will be removed - for (NameValue nameValue : patch.getNameValues()) { - DBElement dbElement = PatchUtil.getColumnNameMap().get(nameValue.getName()); - if (dbElement != null) { - // basePatch.processMultipleNameValues(patch.getNameValues()) - if (!Patch.patch(patch.getId(), patch.getUser(), nameValue, dbElement)) { - throw new RuntimeException("An error occurred while attempting to patch "); - } - if (hasQuestion(nameValue)) { - sendNotificationEmailAndUpdateStatus(patch, nameValues, nameValue, dbElement); - } - controlWorkflowByEmail(patch, nameValue, ddpInstance, profile); - if (patch.getActions() != null) { - writeESWorkflowElseTriggerParticipantEvent(patch, ddpInstance, profile, nameValue); - } - } - else { - throw new RuntimeException("DBElement not found in ColumnNameMap: " + nameValue.getName()); - } - } - return new Result(200, gson.toJson(nameValues)); - } - else { - // mr changes - DBElement dbElement = PatchUtil.getColumnNameMap().get(patch.getNameValue().getName()); - if (dbElement != null) { - if (Patch.patch(patch.getId(), patch.getUser(), patch.getNameValue(), dbElement)) { - List nameValues = setWorkflowRelatedFields(patch); - writeDSMRecordsToES(patch, ddpInstance); - //return nameValues with nulls - return new Result(200, gson.toJson(nameValues)); - } - } - else { - throw new RuntimeException("DBElement not found in ColumnNameMap: " + patch.getNameValue().getName()); - } - } - } - else if (StringUtils.isNotBlank(patch.getParent()) && StringUtils.isNotBlank(patch.getParentId())) { - if (Patch.PARTICIPANT_ID.equals(patch.getParent())) { //Abstraction, Medical Record, Onc history related tables changes - if (StringUtils.isNotBlank(patch.getFieldId())) { // medical record abstraction field id +// if (hasPrimaryKey(patch)) { +// //multiple values are changing +// DDPInstance ddpInstance = DDPInstance.getDDPInstance(patch.getRealm()); +// if (isNameValuePairs(patch)) { +// List nameValues = new ArrayList<>(); +// ESProfile profile = ElasticSearchUtil.getParticipantProfileByGuidOrAltPid(ddpInstance.getParticipantIndexES(), patch.getDdpParticipantId()) +// .orElse(null); +// if (profile == null) { +// logger.error("Unable to find ES profile for participant with guid/altpid: {}, continuing w/ patch", patch.getParentId()); +// } +// // List nameValues = processMultipleNameValues() +// // declared nameValues above will be removed +// for (NameValue nameValue : patch.getNameValues()) { +// DBElement dbElement = PatchUtil.getColumnNameMap().get(nameValue.getName()); +// if (dbElement != null) { +// // basePatch.processMultipleNameValues(patch.getNameValues()) +// if (!Patch.patch(patch.getId(), patch.getUser(), nameValue, dbElement)) { +// throw new RuntimeException("An error occurred while attempting to patch "); +// } +// if (hasQuestion(nameValue)) { +// sendNotificationEmailAndUpdateStatus(patch, nameValues, nameValue, dbElement); +// } +// controlWorkflowByEmail(patch, nameValue, ddpInstance, profile); +// if (patch.getActions() != null) { +// writeESWorkflowElseTriggerParticipantEvent(patch, ddpInstance, profile, nameValue); +// } +// } +// else { +// throw new RuntimeException("DBElement not found in ColumnNameMap: " + nameValue.getName()); +// } +// } +// return new Result(200, gson.toJson(nameValues)); +// } +// else { +// // mr changes +// DBElement dbElement = PatchUtil.getColumnNameMap().get(patch.getNameValue().getName()); +// if (dbElement != null) { +// if (Patch.patch(patch.getId(), patch.getUser(), patch.getNameValue(), dbElement)) { +// List nameValues = setWorkflowRelatedFields(patch); +// writeDSMRecordsToES(patch, ddpInstance); +// //return nameValues with nulls +// return new Result(200, gson.toJson(nameValues)); +// } +// } +// else { +// throw new RuntimeException("DBElement not found in ColumnNameMap: " + patch.getNameValue().getName()); +// } +// } +// } + if (isParentWithPrimaryKey(patch)) { + if (isParentParticipantId(patch)) { //Abstraction, Medical Record, Onc history related tables changes + if (isMedicalRecordAbstractionFieldId(patch)) { // medical record abstraction field id //abstraction related change //multiple value if (isNameValuePairs(patch)) { @@ -314,6 +314,19 @@ else if (Patch.DDP_PARTICIPANT_ID.equals(patch.getParent())) { } } + private boolean isMedicalRecordAbstractionFieldId(Patch patch) { + // only medical_record_abstraction_field_id or another field as well + return StringUtils.isNotBlank(patch.getFieldId()); + } + + private boolean isParentParticipantId(Patch patch) { + return Patch.PARTICIPANT_ID.equals(patch.getParent()); + } + + private boolean isParentWithPrimaryKey(Patch patch) { + return StringUtils.isNotBlank(patch.getParent()) && StringUtils.isNotBlank(patch.getParentId()); + } + private boolean hasPrimaryKey(Patch patch) { return StringUtils.isNotBlank(patch.getId()); } From 3b1b27dd74f743bd01f80b8cf43101c6fcd4ad92 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 5 Oct 2021 16:09:43 +0400 Subject: [PATCH 011/405] refactor | add more template and abstract methods to avoid code duplications --- .../dsm/model/patch/AbstractionPatch.java | 18 +++++++--- .../dsm/model/patch/BasePatch.java | 21 +++++++----- .../dsm/model/patch/NullPatch.java | 9 ++--- .../dsm/model/patch/PrimaryKeyPatch.java | 33 +++++++++---------- .../broadinstitute/dsm/route/PatchRoute.java | 1 - 5 files changed, 45 insertions(+), 37 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java index e55b9b01f..4b51784e8 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java @@ -1,11 +1,14 @@ package org.broadinstitute.dsm.model.patch; +import org.broadinstitute.ddp.handlers.util.Result; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.AbstractionWrapper; import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.Patch; import spark.utils.StringUtils; +import java.util.HashMap; +import java.util.Map; import java.util.Optional; public class AbstractionPatch extends BasePatch { @@ -13,23 +16,30 @@ public class AbstractionPatch extends BasePatch { String primaryKeyId; @Override - public Object doPatch() { + protected Object patchNameValuePairs() { return null; } @Override - Object handleSingleNameValue(DBElement dbElement) { + public Object patchNameValuePair() { return null; + + } + + @Override + Object handleSingleNameValue(DBElement dbElement) { + String primaryKeyId = AbstractionWrapper.createNewAbstractionFieldValue(patch.getParentId(), patch.getFieldId(), patch.getUser(), patch.getNameValue(), dbElement); + return Map.of(PRIMARY_KEY_ID, primaryKeyId); } @Override - Optional processEachNameValue(NameValue nameValue, DBElement dbElement) { + Optional processEachNameValue(NameValue nameValue, DBElement dbElement) { if (StringUtils.isBlank(primaryKeyId)) { primaryKeyId = AbstractionWrapper.createNewAbstractionFieldValue(patch.getParentId(), patch.getFieldId(), patch.getUser(), nameValue, dbElement); } if (!Patch.patch(primaryKeyId, patch.getUser(), nameValue, dbElement)) { throw new RuntimeException("An error occurred while attempting to patch "); } - return Optional.empty(); + return Optional.ofNullable(primaryKeyId); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index ce74fe4ec..57a6f0b2a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -34,7 +34,7 @@ import org.slf4j.LoggerFactory; -public abstract class BasePatch { +public abstract class BasePatch { static final Logger logger = LoggerFactory.getLogger(BasePatch.class); @@ -56,10 +56,15 @@ public BasePatch(Patch patch) { prepareNecessaryData(); } - public abstract T doPatch(); + public Object doPatch() { + return isNameValuePairs() ? patchNameValuePairs() : patchNameValuePair(); + } + + protected abstract Object patchNameValuePairs(); + protected abstract Object patchNameValuePair(); - abstract T handleSingleNameValue(DBElement dbElement); + abstract Object handleSingleNameValue(DBElement dbElement); private void prepareNecessaryData() { ddpInstance = DDPInstance.getDDPInstance(patch.getRealm()); @@ -67,8 +72,8 @@ private void prepareNecessaryData() { .orElse(null); } - Optional processSingleNameValue() { - Optional result; + Optional processSingleNameValue() { + Optional result; DBElement dbElement = PatchUtil.getColumnNameMap().get(patch.getNameValue().getName()); if (dbElement != null) { result = Optional.of(handleSingleNameValue(dbElement)); @@ -79,8 +84,8 @@ Optional processSingleNameValue() { return result; } - T processMultipleNameValues() { - List updatedNameValues = new ArrayList<>(); + Object processMultipleNameValues() { + List updatedNameValues = new ArrayList<>(); for (NameValue nameValue : patch.getNameValues()) { DBElement dbElement = PatchUtil.getColumnNameMap().get(nameValue.getName()); if (dbElement != null) { @@ -93,7 +98,7 @@ T processMultipleNameValues() { return updatedNameValues; } - abstract Optional processEachNameValue(NameValue nameValue, DBElement dbElement); + abstract Optional processEachNameValue(NameValue nameValue, DBElement dbElement); protected boolean hasQuestion(NameValue nameValue) { return nameValue.getName().contains("question"); diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java index d98af1958..a501d3bd2 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java @@ -9,12 +9,12 @@ public class NullPatch extends BasePatch { @Override - public Object doPatch() { + public Object patchNameValuePair() { return new Object(); } @Override - Optional processEachNameValue(NameValue nameValue, DBElement dbElement) { + Optional processEachNameValue(NameValue nameValue, DBElement dbElement) { return Optional.empty(); } @@ -22,9 +22,4 @@ Optional processEachNameValue(NameValue nameValue, DBElement dbElemen Object handleSingleNameValue(DBElement dbElement) { return new Object(); } - - @Override - List processMultipleNameValues(List nameValues) { - return List.of(); - } } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java index 8a9368daf..1e02f6853 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java @@ -38,25 +38,24 @@ public PrimaryKeyPatch(Patch patch, NotificationUtil notificationUtil) { } @Override - public Object doPatch() { - if (isNameValuePairs()) { - ESProfile profile = ElasticSearchUtil.getParticipantProfileByGuidOrAltPid(ddpInstance.getParticipantIndexES(), patch.getDdpParticipantId()) - .orElse(null); - if (profile == null) { - logger.error("Unable to find ES profile for participant with guid/altpid: {}, continuing w/ patch", patch.getParentId()); - } - List updatedNameValues = processMultipleNameValues(patch.getNameValues()); - return new Result(200, GSON.toJson(updatedNameValues)); - } - else { - Optional maybeNameValue = processSingleNameValue(); - return maybeNameValue.orElse(null); + public Object patchNameValuePairs() { + ESProfile profile = ElasticSearchUtil.getParticipantProfileByGuidOrAltPid(ddpInstance.getParticipantIndexES(), patch.getDdpParticipantId()) + .orElse(null); + if (profile == null) { + logger.error("Unable to find ES profile for participant with guid/altpid: {}, continuing w/ patch", patch.getParentId()); } + return processMultipleNameValues(); + } + + @Override + public Object patchNameValuePair() { + Optional maybeNameValue = processSingleNameValue(); + return maybeNameValue.orElse(null); } @Override - Optional processEachNameValue(NameValue nameValue, DBElement dbElement) { - Optional maybeUpdatedNameValue = Optional.empty(); + Optional processEachNameValue(NameValue nameValue, DBElement dbElement) { + Optional maybeUpdatedNameValue = Optional.empty(); if (!Patch.patch(patch.getId(), patch.getUser(), nameValue, dbElement)) { throw new RuntimeException("An error occurred while attempting to patch "); } @@ -70,8 +69,8 @@ Optional processEachNameValue(NameValue nameValue, DBElement dbElemen return maybeUpdatedNameValue; } - private Optional sendNotificationEmailAndUpdateStatus(Patch patch, NameValue nameValue, DBElement dbElement) { - Optional maybeUpdatedNameValue = Optional.empty(); + private Optional sendNotificationEmailAndUpdateStatus(Patch patch, NameValue nameValue, DBElement dbElement) { + Optional maybeUpdatedNameValue = Optional.empty(); UserDto userDto = new UserDao().getUserByEmail(patch.getUser()).orElseThrow(); JSONObject jsonObject = new JSONObject(nameValue.getValue().toString()); JSONArray questionArray = new JSONArray(jsonObject.get("questions").toString()); diff --git a/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java b/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java index b73f5c7dd..c6950e826 100644 --- a/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java @@ -28,7 +28,6 @@ import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import org.broadinstitute.dsm.model.patch.BasePatch; import org.broadinstitute.dsm.model.patch.PatchFactory; -import org.broadinstitute.dsm.model.patch.Patchable; import org.broadinstitute.dsm.model.settings.field.FieldSettings; import org.broadinstitute.dsm.security.RequestHandler; import org.broadinstitute.dsm.statics.DBConstants; From 97c64e6ed496f607ed1d22e1f598f8724658ad6d Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 5 Oct 2021 20:00:50 +0400 Subject: [PATCH 012/405] refactor | finalize the design and implementation for AbstractionPatch.java --- .../dsm/model/patch/AbstractionPatch.java | 20 ++++++++++++++---- .../dsm/model/patch/BasePatch.java | 2 +- .../dsm/model/patch/NullPatch.java | 5 +++++ .../dsm/model/patch/PatchFactory.java | 21 +++++++++++++++---- .../dsm/model/patch/PrimaryKeyPatch.java | 1 - 5 files changed, 39 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java index 4b51784e8..38e6974e7 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java @@ -1,6 +1,5 @@ package org.broadinstitute.dsm.model.patch; -import org.broadinstitute.ddp.handlers.util.Result; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.AbstractionWrapper; import org.broadinstitute.dsm.model.NameValue; @@ -8,22 +7,35 @@ import spark.utils.StringUtils; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; public class AbstractionPatch extends BasePatch { String primaryKeyId; + public static final int FIRST_PRIMARY_KEY_ID = 0; + private static final Map NULL_KEY; + + static { + NULL_KEY = new HashMap<>(); + NULL_KEY.put(PRIMARY_KEY_ID, null); + } + + public AbstractionPatch(Patch patch) { + super(patch); + } @Override protected Object patchNameValuePairs() { - return null; + List firstPrimaryKey = processMultipleNameValues(); + return firstPrimaryKey.isEmpty() ? NULL_KEY : firstPrimaryKey.get(FIRST_PRIMARY_KEY_ID); } @Override public Object patchNameValuePair() { - return null; - + Optional maybeMap = processSingleNameValue(); + return maybeMap.orElse(NULL_KEY); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index 57a6f0b2a..9013d0713 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -84,7 +84,7 @@ Optional processSingleNameValue() { return result; } - Object processMultipleNameValues() { + List processMultipleNameValues() { List updatedNameValues = new ArrayList<>(); for (NameValue nameValue : patch.getNameValues()) { DBElement dbElement = PatchUtil.getColumnNameMap().get(nameValue.getName()); diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java index a501d3bd2..352c983f0 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java @@ -8,6 +8,11 @@ public class NullPatch extends BasePatch { + @Override + protected Object patchNameValuePairs() { + return new Object(); + } + @Override public Object patchNameValuePair() { return new Object(); diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java index 24022048f..af2fb9ab4 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java @@ -10,17 +10,30 @@ public static BasePatch makePatch(Patch patch, NotificationUtil notificationUtil BasePatch patcher = new NullPatch(); if (hasPrimaryKey(patch)) { patcher = new PrimaryKeyPatch(patch, notificationUtil); + } else if (isParentWithPrimaryKey(patch)) { + if (isParentParticipantId(patch)) { + if (isMedicalRecordAbstractionFieldId(patch)) { + patcher = new AbstractionPatch(patch); + } + } } - // switch cases here - return patcher; + // switch cases here + return patcher; } private static boolean hasPrimaryKey(Patch patch) { return StringUtils.isNotBlank(patch.getId()); } + private static boolean isParentWithPrimaryKey(Patch patch) { + return StringUtils.isNotBlank(patch.getParent()) && StringUtils.isNotBlank(patch.getParentId()); + } + private static boolean isParentParticipantId(Patch patch) { + return Patch.PARTICIPANT_ID.equals(patch.getParent()); + } - - + private static boolean isMedicalRecordAbstractionFieldId(Patch patch) { + return StringUtils.isNotBlank(patch.getFieldId()); + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java index 1e02f6853..13f2a829d 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java @@ -4,7 +4,6 @@ import java.util.*; import java.util.concurrent.CopyOnWriteArrayList; -import org.broadinstitute.ddp.handlers.util.Result; import org.broadinstitute.dsm.db.DDPInstance; import org.broadinstitute.dsm.db.dao.settings.EventTypeDao; import org.broadinstitute.dsm.db.dao.user.UserDao; From 2f127debfbd431f6e431487f75913ea5a8f762f5 Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 6 Oct 2021 11:08:58 +0400 Subject: [PATCH 013/405] refactor | initialize MedicalRercordPatch.java for handling medical record related patches --- .../dsm/model/patch/AbstractionPatch.java | 1 - .../dsm/model/patch/BasePatch.java | 2 + .../dsm/model/patch/MedicalRecordPatch.java | 80 +++++++++++++++++++ .../broadinstitute/dsm/route/PatchRoute.java | 2 +- 4 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/patch/MedicalRecordPatch.java diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java index 38e6974e7..0559abaed 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java @@ -15,7 +15,6 @@ public class AbstractionPatch extends BasePatch { String primaryKeyId; public static final int FIRST_PRIMARY_KEY_ID = 0; - private static final Map NULL_KEY; static { NULL_KEY = new HashMap<>(); diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index 9013d0713..d52b14881 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -43,6 +43,8 @@ public abstract class BasePatch { protected static final String NAME_VALUE = "NameValue"; protected static final String STATUS = "status"; protected static final Gson GSON = new GsonBuilder().serializeNulls().create(); + protected static Map NULL_KEY; + protected Patch patch; protected ESProfile profile; diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/MedicalRecordPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/MedicalRecordPatch.java new file mode 100644 index 000000000..f2ee70b0a --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/patch/MedicalRecordPatch.java @@ -0,0 +1,80 @@ +package org.broadinstitute.dsm.model.patch; + +import org.broadinstitute.dsm.db.OncHistoryDetail; +import org.broadinstitute.dsm.db.structure.DBElement; +import org.broadinstitute.dsm.model.NameValue; +import org.broadinstitute.dsm.model.Patch; +import org.broadinstitute.dsm.util.MedicalRecordUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +public class MedicalRecordPatch extends BasePatch { + + private Number mrID; + private String oncHistoryDetailId; + Map resultMap; + + static final Logger logger = LoggerFactory.getLogger(MedicalRecordPatch.class); + + + public MedicalRecordPatch(Patch patch) { + super(patch); + } + + { + + } + + static { + NULL_KEY = new HashMap<>(); + NULL_KEY.put(NAME_VALUE, null); + } + + private void checkBeforePatch() { + mrID = MedicalRecordUtil.isInstitutionTypeInDB(patch.getParentId()); + if (mrID == null) { + // mr of that type doesn't exist yet, so create an institution and mr + MedicalRecordUtil.writeInstitutionIntoDb(patch.getParentId(), MedicalRecordUtil.NOT_SPECIFIED); + mrID = MedicalRecordUtil.isInstitutionTypeInDB(patch.getParentId()); + } + if (mrID != null) { + oncHistoryDetailId = OncHistoryDetail.createNewOncHistoryDetail(mrID.toString(), patch.getUser()); + } + } + + @Override + protected Object patchNameValuePairs() { + checkBeforePatch(); + if (mrID == null) { + logger.error("No medical record id for oncHistoryDetails "); + return NULL_KEY; + } + processMultipleNameValues(); + Map + + return null; + } + + @Override + protected Object patchNameValuePair() { + return null; + } + + @Override + Object handleSingleNameValue(DBElement dbElement) { + return null; + } + + @Override + Optional processEachNameValue(NameValue nameValue, DBElement dbElement) { + if (!Patch.patch(oncHistoryDetailId, patch.getUser(), nameValue, dbElement)) { + throw new RuntimeException("An error occurred while attempting to patch "); + } + return Optional.empty(); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java b/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java index c6950e826..712c2e928 100644 --- a/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java @@ -191,7 +191,7 @@ public Object processRequest(Request request, Response response, String userId) DBElement dbElement = patchUtil.getColumnNameMap().get(nameValue.getName()); if (dbElement != null) { if (!Patch.patch(oncHistoryDetailId, patch.getUser(), nameValue, dbElement)) { - return new RuntimeException("An error occurred while attempting to patch "); + throw new RuntimeException("An error occurred while attempting to patch "); } } else { From e1f5da75af46652d330c9b242dda90479ad5b38d Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 6 Oct 2021 11:40:00 +0400 Subject: [PATCH 014/405] refactor | finished MedicalRecordPatch, start TissuePatch --- .../dsm/model/patch/BasePatch.java | 2 ++ .../dsm/model/patch/MedicalRecordPatch.java | 33 ++++++++++++++---- .../dsm/model/patch/PatchFactory.java | 6 ++-- .../dsm/model/patch/TissuePatch.java | 34 +++++++++++++++++++ 4 files changed, 67 insertions(+), 8 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index d52b14881..94f9a853c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -62,6 +62,8 @@ public Object doPatch() { return isNameValuePairs() ? patchNameValuePairs() : patchNameValuePair(); } + + protected abstract Object patchNameValuePairs(); protected abstract Object patchNameValuePair(); diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/MedicalRecordPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/MedicalRecordPatch.java index f2ee70b0a..36f9ee55c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/MedicalRecordPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/MedicalRecordPatch.java @@ -1,13 +1,16 @@ package org.broadinstitute.dsm.model.patch; +import org.broadinstitute.dsm.db.OncHistory; import org.broadinstitute.dsm.db.OncHistoryDetail; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.Patch; +import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.util.MedicalRecordUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -15,9 +18,12 @@ public class MedicalRecordPatch extends BasePatch { + private static final String ONC_HISTORY_DETAIL_ID = "oncHistoryDetailId"; + private Number mrID; private String oncHistoryDetailId; Map resultMap; + List nameValues; static final Logger logger = LoggerFactory.getLogger(MedicalRecordPatch.class); @@ -27,7 +33,10 @@ public MedicalRecordPatch(Patch patch) { } { - + resultMap = new HashMap<>(); + nameValues = new ArrayList<>(); + nameValues.add(new NameValue("request", OncHistoryDetail.STATUS_REVIEW)); + resultMap.put(NAME_VALUE, GSON.toJson(nameValues)); } static { @@ -45,6 +54,7 @@ private void checkBeforePatch() { if (mrID != null) { oncHistoryDetailId = OncHistoryDetail.createNewOncHistoryDetail(mrID.toString(), patch.getUser()); } + resultMap.put(ONC_HISTORY_DETAIL_ID, oncHistoryDetailId); } @Override @@ -55,19 +65,30 @@ protected Object patchNameValuePairs() { return NULL_KEY; } processMultipleNameValues(); - Map - - return null; + return resultMap; } @Override protected Object patchNameValuePair() { - return null; + checkBeforePatch(); + Optional maybeSingleNameValue = processSingleNameValue(); + return maybeSingleNameValue.orElse(resultMap); } @Override Object handleSingleNameValue(DBElement dbElement) { - return null; + if (Patch.patch(oncHistoryDetailId, patch.getUser(), patch.getNameValue(), dbElement)) { + nameValues.addAll(setWorkflowRelatedFields(patch)); + //set oncHistoryDetails created if it is a oncHistoryDetails value without a ID, otherwise created should already be set + if (dbElement.getTableName().equals(DBConstants.DDP_ONC_HISTORY_DETAIL)) { + NameValue oncHistoryCreated = OncHistory.setOncHistoryCreated(patch.getParentId(), patch.getUser()); + if (oncHistoryCreated.getValue() != null) { + nameValues.add(oncHistoryCreated); + } + } + } + resultMap.put(NAME_VALUE, GSON.toJson(nameValues)); + return resultMap; } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java index af2fb9ab4..13acf083d 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java @@ -14,11 +14,13 @@ public static BasePatch makePatch(Patch patch, NotificationUtil notificationUtil if (isParentParticipantId(patch)) { if (isMedicalRecordAbstractionFieldId(patch)) { patcher = new AbstractionPatch(patch); + } else { + patcher = new MedicalRecordPatch(patch); } } } - // switch cases here - return patcher; + // switch cases here + return patcher; } private static boolean hasPrimaryKey(Patch patch) { diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java new file mode 100644 index 000000000..ecefb630e --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java @@ -0,0 +1,34 @@ +package org.broadinstitute.dsm.model.patch; + +import java.util.Optional; + +import org.broadinstitute.dsm.db.structure.DBElement; +import org.broadinstitute.dsm.model.NameValue; +import org.broadinstitute.dsm.model.Patch; + +public class TissuePatch extends BasePatch { + + public TissuePatch(Patch patch) { + super(patch); + } + + @Override + protected Object patchNameValuePairs() { + return null; + } + + @Override + protected Object patchNameValuePair() { + return null; + } + + @Override + Object handleSingleNameValue(DBElement dbElement) { + return null; + } + + @Override + Optional processEachNameValue(NameValue nameValue, DBElement dbElement) { + return Optional.empty(); + } +} From 64eca2a2a060427824bab10048de5dfeb0fc4cb1 Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 6 Oct 2021 16:00:50 +0400 Subject: [PATCH 015/405] refactor | initialize TissuePatch.java --- .../dsm/model/patch/BasePatch.java | 4 +- .../dsm/model/patch/MedicalRecordPatch.java | 7 +- .../dsm/model/patch/Patchable.java | 5 - .../dsm/model/patch/TissuePatch.java | 7 + .../broadinstitute/dsm/route/PatchRoute.java | 200 +++++++++--------- 5 files changed, 115 insertions(+), 108 deletions(-) delete mode 100644 src/main/java/org/broadinstitute/dsm/model/patch/Patchable.java diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index 94f9a853c..dc01c380f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -55,7 +55,7 @@ public BasePatch() { public BasePatch(Patch patch) { this.patch = patch; - prepareNecessaryData(); + prepareCommonData(); } public Object doPatch() { @@ -70,7 +70,7 @@ public Object doPatch() { abstract Object handleSingleNameValue(DBElement dbElement); - private void prepareNecessaryData() { + private void prepareCommonData() { ddpInstance = DDPInstance.getDDPInstance(patch.getRealm()); profile = ElasticSearchUtil.getParticipantProfileByGuidOrAltPid(ddpInstance.getParticipantIndexES(), patch.getDdpParticipantId()) .orElse(null); diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/MedicalRecordPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/MedicalRecordPatch.java index 36f9ee55c..cf5e9db2b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/MedicalRecordPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/MedicalRecordPatch.java @@ -44,7 +44,7 @@ public MedicalRecordPatch(Patch patch) { NULL_KEY.put(NAME_VALUE, null); } - private void checkBeforePatch() { + private void prepare() { mrID = MedicalRecordUtil.isInstitutionTypeInDB(patch.getParentId()); if (mrID == null) { // mr of that type doesn't exist yet, so create an institution and mr @@ -59,7 +59,7 @@ private void checkBeforePatch() { @Override protected Object patchNameValuePairs() { - checkBeforePatch(); + prepare(); if (mrID == null) { logger.error("No medical record id for oncHistoryDetails "); return NULL_KEY; @@ -70,7 +70,7 @@ protected Object patchNameValuePairs() { @Override protected Object patchNameValuePair() { - checkBeforePatch(); + prepare(); Optional maybeSingleNameValue = processSingleNameValue(); return maybeSingleNameValue.orElse(resultMap); } @@ -98,4 +98,5 @@ Optional processEachNameValue(NameValue nameValue, DBElement dbElement) } return Optional.empty(); } + } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/Patchable.java b/src/main/java/org/broadinstitute/dsm/model/patch/Patchable.java deleted file mode 100644 index 75eded389..000000000 --- a/src/main/java/org/broadinstitute/dsm/model/patch/Patchable.java +++ /dev/null @@ -1,5 +0,0 @@ -package org.broadinstitute.dsm.model.patch; - -public interface Patchable { - -} diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java index ecefb630e..f5f9e92bf 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java @@ -2,12 +2,19 @@ import java.util.Optional; +import org.broadinstitute.dsm.db.Tissue; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.Patch; public class TissuePatch extends BasePatch { + private String tissueId; + + private void prepare() { + tissueId = Tissue.createNewTissue(patch.getParentId(), patch.getUser()); + } + public TissuePatch(Patch patch) { super(patch); } diff --git a/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java b/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java index 712c2e928..735f58164 100644 --- a/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java @@ -133,107 +133,107 @@ public Object processRequest(Request request, Response response, String userId) // } if (isParentWithPrimaryKey(patch)) { if (isParentParticipantId(patch)) { //Abstraction, Medical Record, Onc history related tables changes - if (isMedicalRecordAbstractionFieldId(patch)) { // medical record abstraction field id - //abstraction related change - //multiple value - if (isNameValuePairs(patch)) { - String primaryKeyId = null; - for (NameValue nameValue : patch.getNameValues()) { - DBElement dbElement = PatchUtil.getColumnNameMap().get(nameValue.getName()); - if (dbElement != null) { - if (primaryKeyId == null) { - primaryKeyId = AbstractionWrapper.createNewAbstractionFieldValue(patch.getParentId(), patch.getFieldId(), patch.getUser(), nameValue, dbElement); - } - if (!Patch.patch(primaryKeyId, patch.getUser(), nameValue, dbElement)) { - throw new RuntimeException("An error occurred while attempting to patch "); - } - } - else { - throw new RuntimeException("DBElement not found in ColumnNameMap: " + nameValue.getName()); - } - } - Map map = new HashMap<>(); - map.put(PRIMARY_KEY_ID, primaryKeyId); - //return map with nulls - return new Result(200, gson.toJson(map)); - } - else { - //single value - DBElement dbElement = PatchUtil.getColumnNameMap().get(patch.getNameValue().getName()); - if (dbElement != null) { - String primaryKeyId = AbstractionWrapper.createNewAbstractionFieldValue(patch.getParentId(), patch.getFieldId(), patch.getUser(), patch.getNameValue(), dbElement); - Map map = new HashMap<>(); - map.put(PRIMARY_KEY_ID, primaryKeyId); - //return map with nulls - return new Result(200, gson.toJson(map)); - } - else { - throw new RuntimeException("DBElement not found in ColumnNameMap: " + patch.getNameValue().getName()); - } - } - } - else { - //medical record tracking related change - Number mrID = MedicalRecordUtil.isInstitutionTypeInDB(patch.getParentId()); - if (mrID == null) { - // mr of that type doesn't exist yet, so create an institution and mr - MedicalRecordUtil.writeInstitutionIntoDb(patch.getParentId(), MedicalRecordUtil.NOT_SPECIFIED); - mrID = MedicalRecordUtil.isInstitutionTypeInDB(patch.getParentId()); - } - if (mrID != null) { - // a mr of that type already exits, add oncHistoryDetails to it - String oncHistoryDetailId = OncHistoryDetail.createNewOncHistoryDetail(mrID.toString(), patch.getUser()); - List nameValues = null; - Map map = new HashMap<>(); - //facility was added - if (isNameValuePairs(patch)) { - for (NameValue nameValue : patch.getNameValues()) { - DBElement dbElement = patchUtil.getColumnNameMap().get(nameValue.getName()); - if (dbElement != null) { - if (!Patch.patch(oncHistoryDetailId, patch.getUser(), nameValue, dbElement)) { - throw new RuntimeException("An error occurred while attempting to patch "); - } - } - else { - throw new RuntimeException("DBElement not found in ColumnNameMap: " + nameValue.getName()); - } - } - } - else { - DBElement dbElement = patchUtil.getColumnNameMap().get(patch.getNameValue().getName()); - if (dbElement != null) { - if (Patch.patch(oncHistoryDetailId, patch.getUser(), patch.getNameValue(), dbElement)) { - nameValues = setWorkflowRelatedFields(patch); - //set oncHistoryDetails created if it is a oncHistoryDetails value without a ID, otherwise created should already be set - if (dbElement.getTableName().equals(DBConstants.DDP_ONC_HISTORY_DETAIL)) { - NameValue oncHistoryCreated = OncHistory.setOncHistoryCreated(patch.getParentId(), patch.getUser()); - if (oncHistoryCreated != null && oncHistoryCreated.getValue() != null) { - nameValues.add(oncHistoryCreated); - } - } - } - } - else { - throw new RuntimeException("DBElement not found in ColumnNameMap: " + patch.getNameValue().getName()); - } - } - - // add oncHistoryId and NameValues of objects changed by workflow to json and sent it back to UI - map.put("oncHistoryDetailId", oncHistoryDetailId); - nameValues.add(new NameValue("request", OncHistoryDetail.STATUS_REVIEW)); - map.put(NAME_VALUE, gson.toJson(nameValues)); - //return map with nulls - return new Result(200, gson.toJson(map)); - } - else { - logger.error("No medical record id for oncHistoryDetails "); - } - } +// if (isMedicalRecordAbstractionFieldId(patch)) { // medical record abstraction field id +// //abstraction related change +// //multiple value +// if (isNameValuePairs(patch)) { +// String primaryKeyId = null; +// for (NameValue nameValue : patch.getNameValues()) { +// DBElement dbElement = PatchUtil.getColumnNameMap().get(nameValue.getName()); +// if (dbElement != null) { +// if (primaryKeyId == null) { +// primaryKeyId = AbstractionWrapper.createNewAbstractionFieldValue(patch.getParentId(), patch.getFieldId(), patch.getUser(), nameValue, dbElement); +// } +// if (!Patch.patch(primaryKeyId, patch.getUser(), nameValue, dbElement)) { +// throw new RuntimeException("An error occurred while attempting to patch "); +// } +// } +// else { +// throw new RuntimeException("DBElement not found in ColumnNameMap: " + nameValue.getName()); +// } +// } +// Map map = new HashMap<>(); +// map.put(PRIMARY_KEY_ID, primaryKeyId); +// //return map with nulls +// return new Result(200, gson.toJson(map)); +// } +// else { +// //single value +// DBElement dbElement = PatchUtil.getColumnNameMap().get(patch.getNameValue().getName()); +// if (dbElement != null) { +// String primaryKeyId = AbstractionWrapper.createNewAbstractionFieldValue(patch.getParentId(), patch.getFieldId(), patch.getUser(), patch.getNameValue(), dbElement); +// Map map = new HashMap<>(); +// map.put(PRIMARY_KEY_ID, primaryKeyId); +// //return map with nulls +// return new Result(200, gson.toJson(map)); +// } +// else { +// throw new RuntimeException("DBElement not found in ColumnNameMap: " + patch.getNameValue().getName()); +// } +// } +// } +// else { +// //medical record tracking related change +// Number mrID = MedicalRecordUtil.isInstitutionTypeInDB(patch.getParentId()); +// if (mrID == null) { +// // mr of that type doesn't exist yet, so create an institution and mr +// MedicalRecordUtil.writeInstitutionIntoDb(patch.getParentId(), MedicalRecordUtil.NOT_SPECIFIED); +// mrID = MedicalRecordUtil.isInstitutionTypeInDB(patch.getParentId()); +// } +// if (mrID != null) { +// // a mr of that type already exits, add oncHistoryDetails to it +// String oncHistoryDetailId = OncHistoryDetail.createNewOncHistoryDetail(mrID.toString(), patch.getUser()); +// List nameValues = null; +// Map map = new HashMap<>(); +// //facility was added +// if (isNameValuePairs(patch)) { +// for (NameValue nameValue : patch.getNameValues()) { +// DBElement dbElement = patchUtil.getColumnNameMap().get(nameValue.getName()); +// if (dbElement != null) { +// if (!Patch.patch(oncHistoryDetailId, patch.getUser(), nameValue, dbElement)) { +// throw new RuntimeException("An error occurred while attempting to patch "); +// } +// } +// else { +// throw new RuntimeException("DBElement not found in ColumnNameMap: " + nameValue.getName()); +// } +// } +// } +// else { +// DBElement dbElement = patchUtil.getColumnNameMap().get(patch.getNameValue().getName()); +// if (dbElement != null) { +// if (Patch.patch(oncHistoryDetailId, patch.getUser(), patch.getNameValue(), dbElement)) { +// nameValues = setWorkflowRelatedFields(patch); +// //set oncHistoryDetails created if it is a oncHistoryDetails value without a ID, otherwise created should already be set +// if (dbElement.getTableName().equals(DBConstants.DDP_ONC_HISTORY_DETAIL)) { +// NameValue oncHistoryCreated = OncHistory.setOncHistoryCreated(patch.getParentId(), patch.getUser()); +// if (oncHistoryCreated != null && oncHistoryCreated.getValue() != null) { +// nameValues.add(oncHistoryCreated); +// } +// } +// } +// } +// else { +// throw new RuntimeException("DBElement not found in ColumnNameMap: " + patch.getNameValue().getName()); +// } +// } +// +// // add oncHistoryId and NameValues of objects changed by workflow to json and sent it back to UI +// map.put("oncHistoryDetailId", oncHistoryDetailId); +// nameValues.add(new NameValue("request", OncHistoryDetail.STATUS_REVIEW)); +// map.put(NAME_VALUE, gson.toJson(nameValues)); +// //return map with nulls +// return new Result(200, gson.toJson(map)); +// } +// else { +// logger.error("No medical record id for oncHistoryDetails "); +// } +// } } - else if (Patch.ONC_HISTORY_ID.equals(patch.getParent())) { + else if (isTissueRelatedOncHistoryId(patch)) { // ddp_tissue table related change String tissueId = Tissue.createNewTissue(patch.getParentId(), patch.getUser()); - DBElement dbElement = patchUtil.getColumnNameMap().get(patch.getNameValue().getName()); + DBElement dbElement = PatchUtil.getColumnNameMap().get(patch.getNameValue().getName()); if (dbElement != null) { if (Patch.patch(tissueId, patch.getUser(), patch.getNameValue(), dbElement)) { List nameValues = setWorkflowRelatedFields(patch); @@ -313,6 +313,10 @@ else if (Patch.DDP_PARTICIPANT_ID.equals(patch.getParent())) { } } + private boolean isTissueRelatedOncHistoryId(Patch patch) { + return Patch.ONC_HISTORY_ID.equals(patch.getParent()); + } + private boolean isMedicalRecordAbstractionFieldId(Patch patch) { // only medical_record_abstraction_field_id or another field as well return StringUtils.isNotBlank(patch.getFieldId()); From 757f8bd0ae5e45dd4931792b7f9a3f311de355a9 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 6 Oct 2021 16:44:26 +0400 Subject: [PATCH 016/405] refactor | complete TissuePatch --- .../dsm/model/patch/AbstractionPatch.java | 5 + .../dsm/model/patch/BasePatch.java | 21 +- .../dsm/model/patch/MedicalRecordPatch.java | 9 +- .../dsm/model/patch/NullPatch.java | 5 + .../dsm/model/patch/PatchFactory.java | 4 + .../dsm/model/patch/PrimaryKeyPatch.java | 5 + .../dsm/model/patch/TissuePatch.java | 23 ++- .../broadinstitute/dsm/route/PatchRoute.java | 192 +++++++++--------- 8 files changed, 154 insertions(+), 110 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java index 0559abaed..068c47fb5 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java @@ -25,6 +25,11 @@ public AbstractionPatch(Patch patch) { super(patch); } + @Override + public Object doPatch() { + return isNameValuePairs() ? patchNameValuePairs() : patchNameValuePair(); + } + @Override protected Object patchNameValuePairs() { List firstPrimaryKey = processMultipleNameValues(); diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index dc01c380f..45e782adb 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -3,6 +3,7 @@ import static org.broadinstitute.ddp.db.TransactionWrapper.inTransaction; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -45,25 +46,29 @@ public abstract class BasePatch { protected static final Gson GSON = new GsonBuilder().serializeNulls().create(); protected static Map NULL_KEY; + Map resultMap; + List nameValues; + protected Patch patch; protected ESProfile profile; protected DDPInstance ddpInstance; - public BasePatch() { + { + resultMap = new HashMap<>(); + nameValues = new ArrayList<>(); } - public BasePatch(Patch patch) { - this.patch = patch; - prepareCommonData(); + protected BasePatch() { } - public Object doPatch() { - return isNameValuePairs() ? patchNameValuePairs() : patchNameValuePair(); + protected BasePatch(Patch patch) { + this.patch = patch; + prepareCommonData(); } - - + public abstract Object doPatch(); + protected abstract Object patchNameValuePairs(); protected abstract Object patchNameValuePair(); diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/MedicalRecordPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/MedicalRecordPatch.java index cf5e9db2b..1cf61b457 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/MedicalRecordPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/MedicalRecordPatch.java @@ -22,8 +22,6 @@ public class MedicalRecordPatch extends BasePatch { private Number mrID; private String oncHistoryDetailId; - Map resultMap; - List nameValues; static final Logger logger = LoggerFactory.getLogger(MedicalRecordPatch.class); @@ -32,9 +30,12 @@ public MedicalRecordPatch(Patch patch) { super(patch); } + @Override + public Object doPatch() { + return isNameValuePairs() ? patchNameValuePairs() : patchNameValuePair(); + } + { - resultMap = new HashMap<>(); - nameValues = new ArrayList<>(); nameValues.add(new NameValue("request", OncHistoryDetail.STATUS_REVIEW)); resultMap.put(NAME_VALUE, GSON.toJson(nameValues)); } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java index 352c983f0..765130456 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java @@ -8,6 +8,11 @@ public class NullPatch extends BasePatch { + @Override + public Object doPatch() { + return new Object(); + } + @Override protected Object patchNameValuePairs() { return new Object(); diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java index 13acf083d..98c82c58c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java @@ -38,4 +38,8 @@ private static boolean isParentParticipantId(Patch patch) { private static boolean isMedicalRecordAbstractionFieldId(Patch patch) { return StringUtils.isNotBlank(patch.getFieldId()); } + + private static boolean isTissueRelatedOncHistoryId(Patch patch) { + return Patch.ONC_HISTORY_ID.equals(patch.getParent()); + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java index 13f2a829d..da558ae6f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java @@ -36,6 +36,11 @@ public PrimaryKeyPatch(Patch patch, NotificationUtil notificationUtil) { this.notificationUtil = notificationUtil; } + @Override + public Object doPatch() { + return isNameValuePairs() ? patchNameValuePairs() : patchNameValuePair(); + } + @Override public Object patchNameValuePairs() { ESProfile profile = ElasticSearchUtil.getParticipantProfileByGuidOrAltPid(ddpInstance.getParticipantIndexES(), patch.getDdpParticipantId()) diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java index f5f9e92bf..fb2100652 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java @@ -1,7 +1,11 @@ package org.broadinstitute.dsm.model.patch; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.Optional; +import org.broadinstitute.ddp.handlers.util.Result; import org.broadinstitute.dsm.db.Tissue; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; @@ -9,6 +13,7 @@ public class TissuePatch extends BasePatch { + public static final String TISSUE_ID = "tissueId"; private String tissueId; private void prepare() { @@ -19,6 +24,11 @@ public TissuePatch(Patch patch) { super(patch); } + @Override + public Object doPatch() { + return patchNameValuePair(); + } + @Override protected Object patchNameValuePairs() { return null; @@ -26,12 +36,21 @@ protected Object patchNameValuePairs() { @Override protected Object patchNameValuePair() { - return null; + prepare(); + Optional maybeNameValue = processSingleNameValue(); + return maybeNameValue.orElse(resultMap); } @Override Object handleSingleNameValue(DBElement dbElement) { - return null; + if (Patch.patch(tissueId, patch.getUser(), patch.getNameValue(), dbElement)) { + nameValues = setWorkflowRelatedFields(patch); + resultMap.put(TISSUE_ID, tissueId); + if (!nameValues.isEmpty()) { + resultMap.put(NAME_VALUE, GSON.toJson(nameValues)); + } + } + return resultMap; } @Override diff --git a/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java b/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java index 735f58164..aef27189a 100644 --- a/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java @@ -133,102 +133,102 @@ public Object processRequest(Request request, Response response, String userId) // } if (isParentWithPrimaryKey(patch)) { if (isParentParticipantId(patch)) { //Abstraction, Medical Record, Onc history related tables changes -// if (isMedicalRecordAbstractionFieldId(patch)) { // medical record abstraction field id -// //abstraction related change -// //multiple value -// if (isNameValuePairs(patch)) { -// String primaryKeyId = null; -// for (NameValue nameValue : patch.getNameValues()) { -// DBElement dbElement = PatchUtil.getColumnNameMap().get(nameValue.getName()); -// if (dbElement != null) { -// if (primaryKeyId == null) { -// primaryKeyId = AbstractionWrapper.createNewAbstractionFieldValue(patch.getParentId(), patch.getFieldId(), patch.getUser(), nameValue, dbElement); -// } -// if (!Patch.patch(primaryKeyId, patch.getUser(), nameValue, dbElement)) { -// throw new RuntimeException("An error occurred while attempting to patch "); -// } -// } -// else { -// throw new RuntimeException("DBElement not found in ColumnNameMap: " + nameValue.getName()); -// } -// } -// Map map = new HashMap<>(); -// map.put(PRIMARY_KEY_ID, primaryKeyId); -// //return map with nulls -// return new Result(200, gson.toJson(map)); -// } -// else { -// //single value -// DBElement dbElement = PatchUtil.getColumnNameMap().get(patch.getNameValue().getName()); -// if (dbElement != null) { -// String primaryKeyId = AbstractionWrapper.createNewAbstractionFieldValue(patch.getParentId(), patch.getFieldId(), patch.getUser(), patch.getNameValue(), dbElement); -// Map map = new HashMap<>(); -// map.put(PRIMARY_KEY_ID, primaryKeyId); -// //return map with nulls -// return new Result(200, gson.toJson(map)); -// } -// else { -// throw new RuntimeException("DBElement not found in ColumnNameMap: " + patch.getNameValue().getName()); -// } -// } -// } -// else { -// //medical record tracking related change -// Number mrID = MedicalRecordUtil.isInstitutionTypeInDB(patch.getParentId()); -// if (mrID == null) { -// // mr of that type doesn't exist yet, so create an institution and mr -// MedicalRecordUtil.writeInstitutionIntoDb(patch.getParentId(), MedicalRecordUtil.NOT_SPECIFIED); -// mrID = MedicalRecordUtil.isInstitutionTypeInDB(patch.getParentId()); -// } -// if (mrID != null) { -// // a mr of that type already exits, add oncHistoryDetails to it -// String oncHistoryDetailId = OncHistoryDetail.createNewOncHistoryDetail(mrID.toString(), patch.getUser()); -// List nameValues = null; -// Map map = new HashMap<>(); -// //facility was added -// if (isNameValuePairs(patch)) { -// for (NameValue nameValue : patch.getNameValues()) { -// DBElement dbElement = patchUtil.getColumnNameMap().get(nameValue.getName()); -// if (dbElement != null) { -// if (!Patch.patch(oncHistoryDetailId, patch.getUser(), nameValue, dbElement)) { -// throw new RuntimeException("An error occurred while attempting to patch "); -// } -// } -// else { -// throw new RuntimeException("DBElement not found in ColumnNameMap: " + nameValue.getName()); -// } -// } -// } -// else { -// DBElement dbElement = patchUtil.getColumnNameMap().get(patch.getNameValue().getName()); -// if (dbElement != null) { -// if (Patch.patch(oncHistoryDetailId, patch.getUser(), patch.getNameValue(), dbElement)) { -// nameValues = setWorkflowRelatedFields(patch); -// //set oncHistoryDetails created if it is a oncHistoryDetails value without a ID, otherwise created should already be set -// if (dbElement.getTableName().equals(DBConstants.DDP_ONC_HISTORY_DETAIL)) { -// NameValue oncHistoryCreated = OncHistory.setOncHistoryCreated(patch.getParentId(), patch.getUser()); -// if (oncHistoryCreated != null && oncHistoryCreated.getValue() != null) { -// nameValues.add(oncHistoryCreated); -// } -// } -// } -// } -// else { -// throw new RuntimeException("DBElement not found in ColumnNameMap: " + patch.getNameValue().getName()); -// } -// } -// -// // add oncHistoryId and NameValues of objects changed by workflow to json and sent it back to UI -// map.put("oncHistoryDetailId", oncHistoryDetailId); -// nameValues.add(new NameValue("request", OncHistoryDetail.STATUS_REVIEW)); -// map.put(NAME_VALUE, gson.toJson(nameValues)); -// //return map with nulls -// return new Result(200, gson.toJson(map)); -// } -// else { -// logger.error("No medical record id for oncHistoryDetails "); -// } -// } + if (isMedicalRecordAbstractionFieldId(patch)) { // medical record abstraction field id + //abstraction related change + //multiple value + if (isNameValuePairs(patch)) { + String primaryKeyId = null; + for (NameValue nameValue : patch.getNameValues()) { + DBElement dbElement = PatchUtil.getColumnNameMap().get(nameValue.getName()); + if (dbElement != null) { + if (primaryKeyId == null) { + primaryKeyId = AbstractionWrapper.createNewAbstractionFieldValue(patch.getParentId(), patch.getFieldId(), patch.getUser(), nameValue, dbElement); + } + if (!Patch.patch(primaryKeyId, patch.getUser(), nameValue, dbElement)) { + throw new RuntimeException("An error occurred while attempting to patch "); + } + } + else { + throw new RuntimeException("DBElement not found in ColumnNameMap: " + nameValue.getName()); + } + } + Map map = new HashMap<>(); + map.put(PRIMARY_KEY_ID, primaryKeyId); + //return map with nulls + return new Result(200, gson.toJson(map)); + } + else { + //single value + DBElement dbElement = PatchUtil.getColumnNameMap().get(patch.getNameValue().getName()); + if (dbElement != null) { + String primaryKeyId = AbstractionWrapper.createNewAbstractionFieldValue(patch.getParentId(), patch.getFieldId(), patch.getUser(), patch.getNameValue(), dbElement); + Map map = new HashMap<>(); + map.put(PRIMARY_KEY_ID, primaryKeyId); + //return map with nulls + return new Result(200, gson.toJson(map)); + } + else { + throw new RuntimeException("DBElement not found in ColumnNameMap: " + patch.getNameValue().getName()); + } + } + } + else { + //medical record tracking related change + Number mrID = MedicalRecordUtil.isInstitutionTypeInDB(patch.getParentId()); + if (mrID == null) { + // mr of that type doesn't exist yet, so create an institution and mr + MedicalRecordUtil.writeInstitutionIntoDb(patch.getParentId(), MedicalRecordUtil.NOT_SPECIFIED); + mrID = MedicalRecordUtil.isInstitutionTypeInDB(patch.getParentId()); + } + if (mrID != null) { + // a mr of that type already exits, add oncHistoryDetails to it + String oncHistoryDetailId = OncHistoryDetail.createNewOncHistoryDetail(mrID.toString(), patch.getUser()); + List nameValues = null; + Map map = new HashMap<>(); + //facility was added + if (isNameValuePairs(patch)) { + for (NameValue nameValue : patch.getNameValues()) { + DBElement dbElement = patchUtil.getColumnNameMap().get(nameValue.getName()); + if (dbElement != null) { + if (!Patch.patch(oncHistoryDetailId, patch.getUser(), nameValue, dbElement)) { + throw new RuntimeException("An error occurred while attempting to patch "); + } + } + else { + throw new RuntimeException("DBElement not found in ColumnNameMap: " + nameValue.getName()); + } + } + } + else { + DBElement dbElement = patchUtil.getColumnNameMap().get(patch.getNameValue().getName()); + if (dbElement != null) { + if (Patch.patch(oncHistoryDetailId, patch.getUser(), patch.getNameValue(), dbElement)) { + nameValues = setWorkflowRelatedFields(patch); + //set oncHistoryDetails created if it is a oncHistoryDetails value without a ID, otherwise created should already be set + if (dbElement.getTableName().equals(DBConstants.DDP_ONC_HISTORY_DETAIL)) { + NameValue oncHistoryCreated = OncHistory.setOncHistoryCreated(patch.getParentId(), patch.getUser()); + if (oncHistoryCreated != null && oncHistoryCreated.getValue() != null) { + nameValues.add(oncHistoryCreated); + } + } + } + } + else { + throw new RuntimeException("DBElement not found in ColumnNameMap: " + patch.getNameValue().getName()); + } + } + + // add oncHistoryId and NameValues of objects changed by workflow to json and sent it back to UI + map.put("oncHistoryDetailId", oncHistoryDetailId); + nameValues.add(new NameValue("request", OncHistoryDetail.STATUS_REVIEW)); + map.put(NAME_VALUE, gson.toJson(nameValues)); + //return map with nulls + return new Result(200, gson.toJson(map)); + } + else { + logger.error("No medical record id for oncHistoryDetails "); + } + } } else if (isTissueRelatedOncHistoryId(patch)) { // ddp_tissue table related change From 669de32cf56fc36adfceb9ef748fde1a7176a48a Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 6 Oct 2021 17:16:59 +0400 Subject: [PATCH 017/405] refactor | initialize and complete ParticipandDataPatch.java in one shot --- .../dsm/model/patch/ParticipantDataPatch.java | 66 ++++ .../dsm/model/patch/PatchFactory.java | 8 + .../broadinstitute/dsm/route/PatchRoute.java | 302 +++++++++--------- 3 files changed, 227 insertions(+), 149 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/patch/ParticipantDataPatch.java diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantDataPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantDataPatch.java new file mode 100644 index 000000000..00552dc69 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantDataPatch.java @@ -0,0 +1,66 @@ +package org.broadinstitute.dsm.model.patch; + +import org.broadinstitute.dsm.db.ParticipantData; +import org.broadinstitute.dsm.db.dao.settings.EventTypeDao; +import org.broadinstitute.dsm.db.structure.DBElement; +import org.broadinstitute.dsm.model.NameValue; +import org.broadinstitute.dsm.model.Patch; +import org.broadinstitute.dsm.model.Value; +import org.broadinstitute.dsm.statics.ESObjectConstants; +import org.broadinstitute.dsm.util.ElasticSearchUtil; + +import java.util.Optional; + +public class ParticipantDataPatch extends BasePatch { + + public ParticipantDataPatch(Patch patch) { + super(patch); + } + + private String participantDataId; + + @Override + public Object doPatch() { + return patchNameValuePairs(); + } + + @Override + protected Object patchNameValuePairs() { + processMultipleNameValues(); + return resultMap; + } + + @Override + protected Object patchNameValuePair() { + return null; + } + + @Override + Object handleSingleNameValue(DBElement dbElement) { + return null; + } + + @Override + Optional processEachNameValue(NameValue nameValue, DBElement dbElement) { + if (participantDataId == null) { + participantDataId = ParticipantData.createNewParticipantData(patch.getParentId(), ddpInstance.getDdpInstanceId(), patch.getFieldId(), String.valueOf(nameValue.getValue()), patch.getUser()); + resultMap.put(ESObjectConstants.PARTICIPANT_DATA_ID, participantDataId); + } + else if (participantDataId != null) { + Patch.patch(participantDataId, patch.getUser(), nameValue, dbElement); + } + if (patch.getActions() != null) { + profile = ElasticSearchUtil.getParticipantProfileByGuidOrAltPid(ddpInstance.getParticipantIndexES(), patch.getParentId()) + .orElseThrow(() -> new RuntimeException("Unable to find ES profile for participant: " + patch.getParentId())); + for (Value action : patch.getActions()) { + if (hasProfileAndESWorkflowType(profile, action)) { + writeESWorkflow(patch, nameValue, action, ddpInstance, profile.getParticipantGuid()); + } + else if (EventTypeDao.EVENT.equals(action.getType())) { + triggerParticipantEvent(ddpInstance, patch, action); + } + } + } + return Optional.empty(); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java index 98c82c58c..db9625fb9 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java @@ -17,6 +17,10 @@ public static BasePatch makePatch(Patch patch, NotificationUtil notificationUtil } else { patcher = new MedicalRecordPatch(patch); } + } else if (isTissueRelatedOncHistoryId(patch)) { + patcher = new TissuePatch(patch); + } else if (isParentParticipandDataId(patch)) { + patcher = new ParticipantDataPatch(patch); } } // switch cases here @@ -42,4 +46,8 @@ private static boolean isMedicalRecordAbstractionFieldId(Patch patch) { private static boolean isTissueRelatedOncHistoryId(Patch patch) { return Patch.ONC_HISTORY_ID.equals(patch.getParent()); } + + private static boolean isParentParticipandDataId(Patch patch) { + return Patch.PARTICIPANT_DATA_ID.equals(patch.getParent()); + } } diff --git a/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java b/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java index aef27189a..1c9aaf6f9 100644 --- a/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java @@ -133,156 +133,156 @@ public Object processRequest(Request request, Response response, String userId) // } if (isParentWithPrimaryKey(patch)) { if (isParentParticipantId(patch)) { //Abstraction, Medical Record, Onc history related tables changes - if (isMedicalRecordAbstractionFieldId(patch)) { // medical record abstraction field id - //abstraction related change - //multiple value - if (isNameValuePairs(patch)) { - String primaryKeyId = null; - for (NameValue nameValue : patch.getNameValues()) { - DBElement dbElement = PatchUtil.getColumnNameMap().get(nameValue.getName()); - if (dbElement != null) { - if (primaryKeyId == null) { - primaryKeyId = AbstractionWrapper.createNewAbstractionFieldValue(patch.getParentId(), patch.getFieldId(), patch.getUser(), nameValue, dbElement); - } - if (!Patch.patch(primaryKeyId, patch.getUser(), nameValue, dbElement)) { - throw new RuntimeException("An error occurred while attempting to patch "); - } - } - else { - throw new RuntimeException("DBElement not found in ColumnNameMap: " + nameValue.getName()); - } - } - Map map = new HashMap<>(); - map.put(PRIMARY_KEY_ID, primaryKeyId); - //return map with nulls - return new Result(200, gson.toJson(map)); - } - else { - //single value - DBElement dbElement = PatchUtil.getColumnNameMap().get(patch.getNameValue().getName()); - if (dbElement != null) { - String primaryKeyId = AbstractionWrapper.createNewAbstractionFieldValue(patch.getParentId(), patch.getFieldId(), patch.getUser(), patch.getNameValue(), dbElement); - Map map = new HashMap<>(); - map.put(PRIMARY_KEY_ID, primaryKeyId); - //return map with nulls - return new Result(200, gson.toJson(map)); - } - else { - throw new RuntimeException("DBElement not found in ColumnNameMap: " + patch.getNameValue().getName()); - } - } - } - else { - //medical record tracking related change - Number mrID = MedicalRecordUtil.isInstitutionTypeInDB(patch.getParentId()); - if (mrID == null) { - // mr of that type doesn't exist yet, so create an institution and mr - MedicalRecordUtil.writeInstitutionIntoDb(patch.getParentId(), MedicalRecordUtil.NOT_SPECIFIED); - mrID = MedicalRecordUtil.isInstitutionTypeInDB(patch.getParentId()); - } - if (mrID != null) { - // a mr of that type already exits, add oncHistoryDetails to it - String oncHistoryDetailId = OncHistoryDetail.createNewOncHistoryDetail(mrID.toString(), patch.getUser()); - List nameValues = null; - Map map = new HashMap<>(); - //facility was added - if (isNameValuePairs(patch)) { - for (NameValue nameValue : patch.getNameValues()) { - DBElement dbElement = patchUtil.getColumnNameMap().get(nameValue.getName()); - if (dbElement != null) { - if (!Patch.patch(oncHistoryDetailId, patch.getUser(), nameValue, dbElement)) { - throw new RuntimeException("An error occurred while attempting to patch "); - } - } - else { - throw new RuntimeException("DBElement not found in ColumnNameMap: " + nameValue.getName()); - } - } - } - else { - DBElement dbElement = patchUtil.getColumnNameMap().get(patch.getNameValue().getName()); - if (dbElement != null) { - if (Patch.patch(oncHistoryDetailId, patch.getUser(), patch.getNameValue(), dbElement)) { - nameValues = setWorkflowRelatedFields(patch); - //set oncHistoryDetails created if it is a oncHistoryDetails value without a ID, otherwise created should already be set - if (dbElement.getTableName().equals(DBConstants.DDP_ONC_HISTORY_DETAIL)) { - NameValue oncHistoryCreated = OncHistory.setOncHistoryCreated(patch.getParentId(), patch.getUser()); - if (oncHistoryCreated != null && oncHistoryCreated.getValue() != null) { - nameValues.add(oncHistoryCreated); - } - } - } - } - else { - throw new RuntimeException("DBElement not found in ColumnNameMap: " + patch.getNameValue().getName()); - } - } - - // add oncHistoryId and NameValues of objects changed by workflow to json and sent it back to UI - map.put("oncHistoryDetailId", oncHistoryDetailId); - nameValues.add(new NameValue("request", OncHistoryDetail.STATUS_REVIEW)); - map.put(NAME_VALUE, gson.toJson(nameValues)); - //return map with nulls - return new Result(200, gson.toJson(map)); - } - else { - logger.error("No medical record id for oncHistoryDetails "); - } - } - } - else if (isTissueRelatedOncHistoryId(patch)) { - // ddp_tissue table related change - String tissueId = Tissue.createNewTissue(patch.getParentId(), patch.getUser()); - DBElement dbElement = PatchUtil.getColumnNameMap().get(patch.getNameValue().getName()); - if (dbElement != null) { - if (Patch.patch(tissueId, patch.getUser(), patch.getNameValue(), dbElement)) { - List nameValues = setWorkflowRelatedFields(patch); - Map map = new HashMap<>(); - map.put("tissueId", tissueId); - if (!nameValues.isEmpty()) { - map.put(NAME_VALUE, gson.toJson(nameValues)); - return new Result(200, gson.toJson(map)); - } - else { - return new Result(200, gson.toJson(map)); - } - } - } - else { - throw new RuntimeException("DBElement not found in ColumnNameMap: " + patch.getNameValue().getName()); - } - } - else if (Patch.PARTICIPANT_DATA_ID.equals(patch.getParent())) { - //ddp_participant_data table - String participantDataId = null; - Map map = new HashMap<>(); - DDPInstance ddpInstance = DDPInstance.getDDPInstance(patch.getRealm()); - for (NameValue nameValue : patch.getNameValues()) { - DBElement dbElement = patchUtil.getColumnNameMap().get(nameValue.getName()); - if (dbElement != null) { - if (participantDataId == null) { - participantDataId = ParticipantData.createNewParticipantData(patch.getParentId(), ddpInstance.getDdpInstanceId(), patch.getFieldId(), String.valueOf(nameValue.getValue()), patch.getUser()); - map.put(ESObjectConstants.PARTICIPANT_DATA_ID, participantDataId); - } - else if (participantDataId != null) { - Patch.patch(participantDataId, patch.getUser(), nameValue, dbElement); - } - if (patch.getActions() != null) { - ESProfile profile = ElasticSearchUtil.getParticipantProfileByGuidOrAltPid(ddpInstance.getParticipantIndexES(), patch.getParentId()) - .orElseThrow(() -> new RuntimeException("Unable to find ES profile for participant: " + patch.getParentId())); - for (Value action : patch.getActions()) { - if (hasProfileAndESWorkflowType(profile, action)) { - writeESWorkflow(patch, nameValue, action, ddpInstance, profile.getParticipantGuid()); - } - else if (EventTypeDao.EVENT.equals(action.getType())) { - triggerParticipantEvent(ddpInstance, patch, action); - } - } - } - } - } - return new Result(200, gson.toJson(map)); +// if (isMedicalRecordAbstractionFieldId(patch)) { // medical record abstraction field id +// //abstraction related change +// //multiple value +// if (isNameValuePairs(patch)) { +// String primaryKeyId = null; +// for (NameValue nameValue : patch.getNameValues()) { +// DBElement dbElement = PatchUtil.getColumnNameMap().get(nameValue.getName()); +// if (dbElement != null) { +// if (primaryKeyId == null) { +// primaryKeyId = AbstractionWrapper.createNewAbstractionFieldValue(patch.getParentId(), patch.getFieldId(), patch.getUser(), nameValue, dbElement); +// } +// if (!Patch.patch(primaryKeyId, patch.getUser(), nameValue, dbElement)) { +// throw new RuntimeException("An error occurred while attempting to patch "); +// } +// } +// else { +// throw new RuntimeException("DBElement not found in ColumnNameMap: " + nameValue.getName()); +// } +// } +// Map map = new HashMap<>(); +// map.put(PRIMARY_KEY_ID, primaryKeyId); +// //return map with nulls +// return new Result(200, gson.toJson(map)); +// } +// else { +// //single value +// DBElement dbElement = PatchUtil.getColumnNameMap().get(patch.getNameValue().getName()); +// if (dbElement != null) { +// String primaryKeyId = AbstractionWrapper.createNewAbstractionFieldValue(patch.getParentId(), patch.getFieldId(), patch.getUser(), patch.getNameValue(), dbElement); +// Map map = new HashMap<>(); +// map.put(PRIMARY_KEY_ID, primaryKeyId); +// //return map with nulls +// return new Result(200, gson.toJson(map)); +// } +// else { +// throw new RuntimeException("DBElement not found in ColumnNameMap: " + patch.getNameValue().getName()); +// } +// } +// } +// else { +// //medical record tracking related change +// Number mrID = MedicalRecordUtil.isInstitutionTypeInDB(patch.getParentId()); +// if (mrID == null) { +// // mr of that type doesn't exist yet, so create an institution and mr +// MedicalRecordUtil.writeInstitutionIntoDb(patch.getParentId(), MedicalRecordUtil.NOT_SPECIFIED); +// mrID = MedicalRecordUtil.isInstitutionTypeInDB(patch.getParentId()); +// } +// if (mrID != null) { +// // a mr of that type already exits, add oncHistoryDetails to it +// String oncHistoryDetailId = OncHistoryDetail.createNewOncHistoryDetail(mrID.toString(), patch.getUser()); +// List nameValues = null; +// Map map = new HashMap<>(); +// //facility was added +// if (isNameValuePairs(patch)) { +// for (NameValue nameValue : patch.getNameValues()) { +// DBElement dbElement = patchUtil.getColumnNameMap().get(nameValue.getName()); +// if (dbElement != null) { +// if (!Patch.patch(oncHistoryDetailId, patch.getUser(), nameValue, dbElement)) { +// throw new RuntimeException("An error occurred while attempting to patch "); +// } +// } +// else { +// throw new RuntimeException("DBElement not found in ColumnNameMap: " + nameValue.getName()); +// } +// } +// } +// else { +// DBElement dbElement = patchUtil.getColumnNameMap().get(patch.getNameValue().getName()); +// if (dbElement != null) { +// if (Patch.patch(oncHistoryDetailId, patch.getUser(), patch.getNameValue(), dbElement)) { +// nameValues = setWorkflowRelatedFields(patch); +// //set oncHistoryDetails created if it is a oncHistoryDetails value without a ID, otherwise created should already be set +// if (dbElement.getTableName().equals(DBConstants.DDP_ONC_HISTORY_DETAIL)) { +// NameValue oncHistoryCreated = OncHistory.setOncHistoryCreated(patch.getParentId(), patch.getUser()); +// if (oncHistoryCreated != null && oncHistoryCreated.getValue() != null) { +// nameValues.add(oncHistoryCreated); +// } +// } +// } +// } +// else { +// throw new RuntimeException("DBElement not found in ColumnNameMap: " + patch.getNameValue().getName()); +// } +// } +// +// // add oncHistoryId and NameValues of objects changed by workflow to json and sent it back to UI +// map.put("oncHistoryDetailId", oncHistoryDetailId); +// nameValues.add(new NameValue("request", OncHistoryDetail.STATUS_REVIEW)); +// map.put(NAME_VALUE, gson.toJson(nameValues)); +// //return map with nulls +// return new Result(200, gson.toJson(map)); +// } +// else { +// logger.error("No medical record id for oncHistoryDetails "); +// } +// } +// } +// else if (isTissueRelatedOncHistoryId(patch)) { +// // ddp_tissue table related change +// String tissueId = Tissue.createNewTissue(patch.getParentId(), patch.getUser()); +// DBElement dbElement = PatchUtil.getColumnNameMap().get(patch.getNameValue().getName()); +// if (dbElement != null) { +// if (Patch.patch(tissueId, patch.getUser(), patch.getNameValue(), dbElement)) { +// List nameValues = setWorkflowRelatedFields(patch); +// Map map = new HashMap<>(); +// map.put("tissueId", tissueId); +// if (!nameValues.isEmpty()) { +// map.put(NAME_VALUE, gson.toJson(nameValues)); +// return new Result(200, gson.toJson(map)); +// } +// else { +// return new Result(200, gson.toJson(map)); +// } +// } +// } +// else { +// throw new RuntimeException("DBElement not found in ColumnNameMap: " + patch.getNameValue().getName()); +// } } +// else if (isParentParticipandDataId(patch)) { +// //ddp_participant_data table +// String participantDataId = null; +// Map map = new HashMap<>(); +// DDPInstance ddpInstance = DDPInstance.getDDPInstance(patch.getRealm()); +// for (NameValue nameValue : patch.getNameValues()) { +// DBElement dbElement = patchUtil.getColumnNameMap().get(nameValue.getName()); +// if (dbElement != null) { +// if (participantDataId == null) { +// participantDataId = ParticipantData.createNewParticipantData(patch.getParentId(), ddpInstance.getDdpInstanceId(), patch.getFieldId(), String.valueOf(nameValue.getValue()), patch.getUser()); +// map.put(ESObjectConstants.PARTICIPANT_DATA_ID, participantDataId); +// } +// else if (participantDataId != null) { +// Patch.patch(participantDataId, patch.getUser(), nameValue, dbElement); +// } +// if (patch.getActions() != null) { +// ESProfile profile = ElasticSearchUtil.getParticipantProfileByGuidOrAltPid(ddpInstance.getParticipantIndexES(), patch.getParentId()) +// .orElseThrow(() -> new RuntimeException("Unable to find ES profile for participant: " + patch.getParentId())); +// for (Value action : patch.getActions()) { +// if (hasProfileAndESWorkflowType(profile, action)) { +// writeESWorkflow(patch, nameValue, action, ddpInstance, profile.getParticipantGuid()); +// } +// else if (EventTypeDao.EVENT.equals(action.getType())) { +// triggerParticipantEvent(ddpInstance, patch, action); +// } +// } +// } +// } +// } +// return new Result(200, gson.toJson(map)); +// } else if (Patch.DDP_PARTICIPANT_ID.equals(patch.getParent())) { //new additional value for pt which is not in ddp_participant and ddp_participant_record table yet DDPInstance ddpInstance = DDPInstance.getDDPInstance(patch.getRealm()); @@ -313,6 +313,10 @@ else if (Patch.DDP_PARTICIPANT_ID.equals(patch.getParent())) { } } + private boolean isParentParticipandDataId(Patch patch) { + return Patch.PARTICIPANT_DATA_ID.equals(patch.getParent()); + } + private boolean isTissueRelatedOncHistoryId(Patch patch) { return Patch.ONC_HISTORY_ID.equals(patch.getParent()); } From 354b04a6e1f271ef3986c941f8580191c9e21fc0 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 6 Oct 2021 18:40:17 +0400 Subject: [PATCH 018/405] refactor | add ParticipantRecordPatch, eliminate old code --- .../dsm/model/patch/MedicalRecordPatch.java | 9 +- .../dsm/model/patch/NullPatch.java | 1 - .../model/patch/ParticipantRecordPatch.java | 77 +++ .../dsm/model/patch/PatchFactory.java | 7 +- .../dsm/model/patch/TissuePatch.java | 4 - .../broadinstitute/dsm/route/PatchRoute.java | 585 +----------------- 6 files changed, 88 insertions(+), 595 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/patch/ParticipantRecordPatch.java diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/MedicalRecordPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/MedicalRecordPatch.java index 1cf61b457..e40a826fa 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/MedicalRecordPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/MedicalRecordPatch.java @@ -1,5 +1,8 @@ package org.broadinstitute.dsm.model.patch; +import java.util.HashMap; +import java.util.Optional; + import org.broadinstitute.dsm.db.OncHistory; import org.broadinstitute.dsm.db.OncHistoryDetail; import org.broadinstitute.dsm.db.structure.DBElement; @@ -10,12 +13,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; - public class MedicalRecordPatch extends BasePatch { private static final String ONC_HISTORY_DETAIL_ID = "oncHistoryDetailId"; diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java index 765130456..f422666d1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java @@ -1,6 +1,5 @@ package org.broadinstitute.dsm.model.patch; -import java.util.List; import java.util.Optional; import org.broadinstitute.dsm.db.structure.DBElement; diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantRecordPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantRecordPatch.java new file mode 100644 index 000000000..2e5a83ad3 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantRecordPatch.java @@ -0,0 +1,77 @@ +package org.broadinstitute.dsm.model.patch; + +import java.util.Optional; + +import org.broadinstitute.dsm.db.DDPInstance; +import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDao; +import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantRecordDao; +import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDto; +import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantRecordDto; +import org.broadinstitute.dsm.db.structure.DBElement; +import org.broadinstitute.dsm.model.NameValue; +import org.broadinstitute.dsm.model.Patch; + +public class ParticipantRecordPatch extends BasePatch { + + private int participantId; + + public ParticipantRecordPatch(Patch patch) { + super(patch); + } + + @Override + public Object doPatch() { + return patchNameValuePair(); + } + + @Override + protected Object patchNameValuePairs() { + return null; + } + + @Override + protected Object patchNameValuePair() { + prepare(); + Optional maybeResultMap = Optional.empty(); + if (participantId > 0) { + maybeResultMap = processSingleNameValue(); + } + return maybeResultMap.orElse(resultMap); + } + + private void prepare() { + participantId = insertDdpParticipant(patch, ddpInstance); + insertDdpParticipantRecord(participantId); + } + + private int insertDdpParticipant(Patch patch, DDPInstance ddpInstance) { + ParticipantDto participantDto = + new ParticipantDto.Builder(Integer.parseInt(ddpInstance.getDdpInstanceId()), System.currentTimeMillis()) + .withDdpParticipantId(patch.getParentId()) + .withLastVersion(0) + .withLastVersionDate("") + .withChangedBy(patch.getUser()) + .build(); + return new ParticipantDao().create(participantDto); + } + + private void insertDdpParticipantRecord(int participantId) { + ParticipantRecordDto participantRecordDto = + new ParticipantRecordDto.Builder(participantId, System.currentTimeMillis()) + .withChangedBy("SYSTEM") + .builder(); + new ParticipantRecordDao().create(participantRecordDto); + } + + @Override + Object handleSingleNameValue(DBElement dbElement) { + Patch.patch(String.valueOf(participantId), patch.getUser(), patch.getNameValue(), dbElement); + resultMap.put(PARTICIPANT_ID, String.valueOf(participantId)); + return resultMap; + } + + @Override + Optional processEachNameValue(NameValue nameValue, DBElement dbElement) { + return Optional.empty(); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java index db9625fb9..478a99b0c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java @@ -21,9 +21,10 @@ public static BasePatch makePatch(Patch patch, NotificationUtil notificationUtil patcher = new TissuePatch(patch); } else if (isParentParticipandDataId(patch)) { patcher = new ParticipantDataPatch(patch); + } else if (isParticipantIdForRecord(patch)) { + patcher = new ParticipantRecordPatch(patch); } } - // switch cases here return patcher; } @@ -50,4 +51,8 @@ private static boolean isTissueRelatedOncHistoryId(Patch patch) { private static boolean isParentParticipandDataId(Patch patch) { return Patch.PARTICIPANT_DATA_ID.equals(patch.getParent()); } + + private static boolean isParticipantIdForRecord(Patch patch) { + return Patch.DDP_PARTICIPANT_ID.equals(patch.getParent()); + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java index fb2100652..b40d37024 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java @@ -1,11 +1,7 @@ package org.broadinstitute.dsm.model.patch; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import java.util.Optional; -import org.broadinstitute.ddp.handlers.util.Result; import org.broadinstitute.dsm.db.Tissue; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; diff --git a/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java b/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java index 1c9aaf6f9..b88cf3152 100644 --- a/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java @@ -2,63 +2,28 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import com.google.gson.reflect.TypeToken; import lombok.NonNull; -import org.apache.commons.lang3.StringUtils; import org.broadinstitute.ddp.handlers.util.Result; -import org.broadinstitute.dsm.db.*; -import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDao; -import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; -import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantRecordDao; -import org.broadinstitute.dsm.db.dao.queue.EventDao; -import org.broadinstitute.dsm.db.dao.settings.EventTypeDao; -import org.broadinstitute.dsm.db.dao.user.UserDao; -import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDto; -import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantRecordDto; -import org.broadinstitute.dsm.db.dto.settings.EventTypeDto; -import org.broadinstitute.dsm.db.dto.user.UserDto; -import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.exception.DuplicateException; -import org.broadinstitute.dsm.export.WorkflowForES; -import org.broadinstitute.dsm.model.AbstractionWrapper; -import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.Patch; -import org.broadinstitute.dsm.model.Value; -import org.broadinstitute.dsm.model.elasticsearch.ESProfile; -import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import org.broadinstitute.dsm.model.patch.BasePatch; import org.broadinstitute.dsm.model.patch.PatchFactory; -import org.broadinstitute.dsm.model.settings.field.FieldSettings; import org.broadinstitute.dsm.security.RequestHandler; import org.broadinstitute.dsm.statics.DBConstants; -import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.statics.UserErrorMessages; import org.broadinstitute.dsm.util.*; -import org.json.JSONArray; -import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import spark.Request; import spark.Response; -import java.io.IOException; -import java.util.*; -import java.util.concurrent.CopyOnWriteArrayList; import static org.broadinstitute.ddp.db.TransactionWrapper.inTransaction; //Class needs to be refactored as soon as possible!!! public class PatchRoute extends RequestHandler { private static final Logger logger = LoggerFactory.getLogger(PatchRoute.class); - private static final ParticipantDataDao participantDataDao = new ParticipantDataDao(); - private static final EventDao eventDao = new EventDao(); - private static final EventTypeDao eventTypeDao = new EventTypeDao(); - private static final Gson gson = new GsonBuilder().serializeNulls().create(); - - private static final String PARTICIPANT_ID = "participantId"; - private static final String PRIMARY_KEY_ID = "primaryKeyId"; - private static final String NAME_VALUE = "NameValue"; - private static final String STATUS = "status"; + private static final Gson GSON = new GsonBuilder().serializeNulls().create(); private NotificationUtil notificationUtil; private PatchUtil patchUtil; @@ -78,226 +43,9 @@ public Object processRequest(Request request, Response response, String userId) || UserUtil.checkUserAccess(null, userId, DBConstants.MR_VIEW, userIdRequest) || UserUtil.checkUserAccess(null, userId, DBConstants.PT_LIST_VIEW, userIdRequest)) { try { String requestBody = request.body(); - Patch patch = gson.fromJson(requestBody, Patch.class); + Patch patch = GSON.fromJson(requestBody, Patch.class); BasePatch patcher = PatchFactory.makePatch(patch, notificationUtil); patcher.doPatch(); - -// if (hasPrimaryKey(patch)) { -// //multiple values are changing -// DDPInstance ddpInstance = DDPInstance.getDDPInstance(patch.getRealm()); -// if (isNameValuePairs(patch)) { -// List nameValues = new ArrayList<>(); -// ESProfile profile = ElasticSearchUtil.getParticipantProfileByGuidOrAltPid(ddpInstance.getParticipantIndexES(), patch.getDdpParticipantId()) -// .orElse(null); -// if (profile == null) { -// logger.error("Unable to find ES profile for participant with guid/altpid: {}, continuing w/ patch", patch.getParentId()); -// } -// // List nameValues = processMultipleNameValues() -// // declared nameValues above will be removed -// for (NameValue nameValue : patch.getNameValues()) { -// DBElement dbElement = PatchUtil.getColumnNameMap().get(nameValue.getName()); -// if (dbElement != null) { -// // basePatch.processMultipleNameValues(patch.getNameValues()) -// if (!Patch.patch(patch.getId(), patch.getUser(), nameValue, dbElement)) { -// throw new RuntimeException("An error occurred while attempting to patch "); -// } -// if (hasQuestion(nameValue)) { -// sendNotificationEmailAndUpdateStatus(patch, nameValues, nameValue, dbElement); -// } -// controlWorkflowByEmail(patch, nameValue, ddpInstance, profile); -// if (patch.getActions() != null) { -// writeESWorkflowElseTriggerParticipantEvent(patch, ddpInstance, profile, nameValue); -// } -// } -// else { -// throw new RuntimeException("DBElement not found in ColumnNameMap: " + nameValue.getName()); -// } -// } -// return new Result(200, gson.toJson(nameValues)); -// } -// else { -// // mr changes -// DBElement dbElement = PatchUtil.getColumnNameMap().get(patch.getNameValue().getName()); -// if (dbElement != null) { -// if (Patch.patch(patch.getId(), patch.getUser(), patch.getNameValue(), dbElement)) { -// List nameValues = setWorkflowRelatedFields(patch); -// writeDSMRecordsToES(patch, ddpInstance); -// //return nameValues with nulls -// return new Result(200, gson.toJson(nameValues)); -// } -// } -// else { -// throw new RuntimeException("DBElement not found in ColumnNameMap: " + patch.getNameValue().getName()); -// } -// } -// } - if (isParentWithPrimaryKey(patch)) { - if (isParentParticipantId(patch)) { //Abstraction, Medical Record, Onc history related tables changes -// if (isMedicalRecordAbstractionFieldId(patch)) { // medical record abstraction field id -// //abstraction related change -// //multiple value -// if (isNameValuePairs(patch)) { -// String primaryKeyId = null; -// for (NameValue nameValue : patch.getNameValues()) { -// DBElement dbElement = PatchUtil.getColumnNameMap().get(nameValue.getName()); -// if (dbElement != null) { -// if (primaryKeyId == null) { -// primaryKeyId = AbstractionWrapper.createNewAbstractionFieldValue(patch.getParentId(), patch.getFieldId(), patch.getUser(), nameValue, dbElement); -// } -// if (!Patch.patch(primaryKeyId, patch.getUser(), nameValue, dbElement)) { -// throw new RuntimeException("An error occurred while attempting to patch "); -// } -// } -// else { -// throw new RuntimeException("DBElement not found in ColumnNameMap: " + nameValue.getName()); -// } -// } -// Map map = new HashMap<>(); -// map.put(PRIMARY_KEY_ID, primaryKeyId); -// //return map with nulls -// return new Result(200, gson.toJson(map)); -// } -// else { -// //single value -// DBElement dbElement = PatchUtil.getColumnNameMap().get(patch.getNameValue().getName()); -// if (dbElement != null) { -// String primaryKeyId = AbstractionWrapper.createNewAbstractionFieldValue(patch.getParentId(), patch.getFieldId(), patch.getUser(), patch.getNameValue(), dbElement); -// Map map = new HashMap<>(); -// map.put(PRIMARY_KEY_ID, primaryKeyId); -// //return map with nulls -// return new Result(200, gson.toJson(map)); -// } -// else { -// throw new RuntimeException("DBElement not found in ColumnNameMap: " + patch.getNameValue().getName()); -// } -// } -// } -// else { -// //medical record tracking related change -// Number mrID = MedicalRecordUtil.isInstitutionTypeInDB(patch.getParentId()); -// if (mrID == null) { -// // mr of that type doesn't exist yet, so create an institution and mr -// MedicalRecordUtil.writeInstitutionIntoDb(patch.getParentId(), MedicalRecordUtil.NOT_SPECIFIED); -// mrID = MedicalRecordUtil.isInstitutionTypeInDB(patch.getParentId()); -// } -// if (mrID != null) { -// // a mr of that type already exits, add oncHistoryDetails to it -// String oncHistoryDetailId = OncHistoryDetail.createNewOncHistoryDetail(mrID.toString(), patch.getUser()); -// List nameValues = null; -// Map map = new HashMap<>(); -// //facility was added -// if (isNameValuePairs(patch)) { -// for (NameValue nameValue : patch.getNameValues()) { -// DBElement dbElement = patchUtil.getColumnNameMap().get(nameValue.getName()); -// if (dbElement != null) { -// if (!Patch.patch(oncHistoryDetailId, patch.getUser(), nameValue, dbElement)) { -// throw new RuntimeException("An error occurred while attempting to patch "); -// } -// } -// else { -// throw new RuntimeException("DBElement not found in ColumnNameMap: " + nameValue.getName()); -// } -// } -// } -// else { -// DBElement dbElement = patchUtil.getColumnNameMap().get(patch.getNameValue().getName()); -// if (dbElement != null) { -// if (Patch.patch(oncHistoryDetailId, patch.getUser(), patch.getNameValue(), dbElement)) { -// nameValues = setWorkflowRelatedFields(patch); -// //set oncHistoryDetails created if it is a oncHistoryDetails value without a ID, otherwise created should already be set -// if (dbElement.getTableName().equals(DBConstants.DDP_ONC_HISTORY_DETAIL)) { -// NameValue oncHistoryCreated = OncHistory.setOncHistoryCreated(patch.getParentId(), patch.getUser()); -// if (oncHistoryCreated != null && oncHistoryCreated.getValue() != null) { -// nameValues.add(oncHistoryCreated); -// } -// } -// } -// } -// else { -// throw new RuntimeException("DBElement not found in ColumnNameMap: " + patch.getNameValue().getName()); -// } -// } -// -// // add oncHistoryId and NameValues of objects changed by workflow to json and sent it back to UI -// map.put("oncHistoryDetailId", oncHistoryDetailId); -// nameValues.add(new NameValue("request", OncHistoryDetail.STATUS_REVIEW)); -// map.put(NAME_VALUE, gson.toJson(nameValues)); -// //return map with nulls -// return new Result(200, gson.toJson(map)); -// } -// else { -// logger.error("No medical record id for oncHistoryDetails "); -// } -// } -// } -// else if (isTissueRelatedOncHistoryId(patch)) { -// // ddp_tissue table related change -// String tissueId = Tissue.createNewTissue(patch.getParentId(), patch.getUser()); -// DBElement dbElement = PatchUtil.getColumnNameMap().get(patch.getNameValue().getName()); -// if (dbElement != null) { -// if (Patch.patch(tissueId, patch.getUser(), patch.getNameValue(), dbElement)) { -// List nameValues = setWorkflowRelatedFields(patch); -// Map map = new HashMap<>(); -// map.put("tissueId", tissueId); -// if (!nameValues.isEmpty()) { -// map.put(NAME_VALUE, gson.toJson(nameValues)); -// return new Result(200, gson.toJson(map)); -// } -// else { -// return new Result(200, gson.toJson(map)); -// } -// } -// } -// else { -// throw new RuntimeException("DBElement not found in ColumnNameMap: " + patch.getNameValue().getName()); -// } - } -// else if (isParentParticipandDataId(patch)) { -// //ddp_participant_data table -// String participantDataId = null; -// Map map = new HashMap<>(); -// DDPInstance ddpInstance = DDPInstance.getDDPInstance(patch.getRealm()); -// for (NameValue nameValue : patch.getNameValues()) { -// DBElement dbElement = patchUtil.getColumnNameMap().get(nameValue.getName()); -// if (dbElement != null) { -// if (participantDataId == null) { -// participantDataId = ParticipantData.createNewParticipantData(patch.getParentId(), ddpInstance.getDdpInstanceId(), patch.getFieldId(), String.valueOf(nameValue.getValue()), patch.getUser()); -// map.put(ESObjectConstants.PARTICIPANT_DATA_ID, participantDataId); -// } -// else if (participantDataId != null) { -// Patch.patch(participantDataId, patch.getUser(), nameValue, dbElement); -// } -// if (patch.getActions() != null) { -// ESProfile profile = ElasticSearchUtil.getParticipantProfileByGuidOrAltPid(ddpInstance.getParticipantIndexES(), patch.getParentId()) -// .orElseThrow(() -> new RuntimeException("Unable to find ES profile for participant: " + patch.getParentId())); -// for (Value action : patch.getActions()) { -// if (hasProfileAndESWorkflowType(profile, action)) { -// writeESWorkflow(patch, nameValue, action, ddpInstance, profile.getParticipantGuid()); -// } -// else if (EventTypeDao.EVENT.equals(action.getType())) { -// triggerParticipantEvent(ddpInstance, patch, action); -// } -// } -// } -// } -// } -// return new Result(200, gson.toJson(map)); -// } - else if (Patch.DDP_PARTICIPANT_ID.equals(patch.getParent())) { - //new additional value for pt which is not in ddp_participant and ddp_participant_record table yet - DDPInstance ddpInstance = DDPInstance.getDDPInstance(patch.getRealm()); - int participantId = insertDdpParticipant(patch, ddpInstance); - insertDdpParticipantRecord(participantId); - if (participantId > 0) { - DBElement dbElement = patchUtil.getColumnNameMap().get(patch.getNameValue().getName()); - if (dbElement == null) { - throw new RuntimeException("DBElement not found in ColumnNameMap: " + patch.getNameValue().getName()); - } - Patch.patch(String.valueOf(participantId), patch.getUser(), patch.getNameValue(), dbElement); - return new Result(200, gson.toJson(Map.of(PARTICIPANT_ID, String.valueOf(participantId)))); - } - } - } throw new RuntimeException("Id and parentId was null"); } catch (DuplicateException e) { @@ -312,333 +60,4 @@ else if (Patch.DDP_PARTICIPANT_ID.equals(patch.getParent())) { return new Result(500, UserErrorMessages.NO_RIGHTS); } } - - private boolean isParentParticipandDataId(Patch patch) { - return Patch.PARTICIPANT_DATA_ID.equals(patch.getParent()); - } - - private boolean isTissueRelatedOncHistoryId(Patch patch) { - return Patch.ONC_HISTORY_ID.equals(patch.getParent()); - } - - private boolean isMedicalRecordAbstractionFieldId(Patch patch) { - // only medical_record_abstraction_field_id or another field as well - return StringUtils.isNotBlank(patch.getFieldId()); - } - - private boolean isParentParticipantId(Patch patch) { - return Patch.PARTICIPANT_ID.equals(patch.getParent()); - } - - private boolean isParentWithPrimaryKey(Patch patch) { - return StringUtils.isNotBlank(patch.getParent()) && StringUtils.isNotBlank(patch.getParentId()); - } - - private boolean hasPrimaryKey(Patch patch) { - return StringUtils.isNotBlank(patch.getId()); - } - - private boolean isNameValuePairs(Patch patch) { - //TODO -> could be changed later after clarification - return patch.getNameValues() != null && !patch.getNameValues().isEmpty(); - } - - private void writeESWorkflowElseTriggerParticipantEvent(Patch patch, DDPInstance ddpInstance, ESProfile profile, NameValue nameValue) { - for (Value action : patch.getActions()) { - if (hasProfileAndESWorkflowType(profile, action)) { - writeESWorkflow(patch, nameValue, action, ddpInstance, profile.getParticipantGuid()); - } - else if (EventTypeDao.EVENT.equals(action.getType())) { - triggerParticipantEvent(ddpInstance, patch, action); - } - } - } - - private boolean hasProfileAndESWorkflowType(ESProfile profile, Value action) { - return ESObjectConstants.ELASTIC_EXPORT_WORKFLOWS.equals(action.getType()) && profile != null; - } - - private boolean hasQuestion(NameValue nameValue) { - return nameValue.getName().contains("question"); - } - - private void sendNotificationEmailAndUpdateStatus(Patch patch, List nameValues, NameValue nameValue, DBElement dbElement) { - UserDto userDto = new UserDao().getUserByEmail(patch.getUser()).orElseThrow(); - JSONObject jsonObject = new JSONObject(nameValue.getValue().toString()); - JSONArray questionArray = new JSONArray(jsonObject.get("questions").toString()); - boolean writeBack = false; - for (int i = 0; i < questionArray.length(); i++) { - JSONObject question = questionArray.getJSONObject(i); - if (isSent(question)) { - if (question.optString("email") != null && question.optString("question") != null) { - notificationUtil.sentAbstractionExpertQuestion(userDto.getEmail().orElse(""), userDto.getName().orElse(""), question.optString("email"), - patch.getFieldName(), question.optString("question"), notificationUtil.getTemplate("DSM_ABSTRACTION_EXPERT_QUESTION")); - } - question.put(STATUS, "done"); - writeBack = true; - } - } - if (writeBack) { - jsonObject.put("questions", questionArray); - String str = jsonObject.toString(); - nameValue.setValue(str); - if (!Patch.patch(patch.getId(), patch.getUser(), nameValue, dbElement)) { - throw new RuntimeException("An error occurred while attempting to patch "); - } - nameValues.add(nameValue); - } - } - - private boolean isSent(JSONObject question) { - return question.optString(STATUS) != null && question.optString(STATUS).equals("sent"); - } - - private void controlWorkflowByEmail(Patch patch, NameValue nameValue, DDPInstance ddpInstance, ESProfile profile) { - if (profile == null || nameValue.getValue() == null) { - return; - } - try { - Map pData = gson.fromJson(nameValue.getValue().toString(), Map.class); - org.broadinstitute.dsm.model.participant.data.ParticipantData participantData = - new org.broadinstitute.dsm.model.participant.data.ParticipantData(Integer.parseInt(patch.getId()), - patch.getParentId(), Integer.parseInt(ddpInstance.getDdpInstanceId()), patch.getFieldId(), - pData); - - if (participantData.hasFamilyMemberApplicantEmail(profile)) { - writeFamilyMemberWorklow(patch, ddpInstance, profile, pData); - } else { - Map esMap = ElasticSearchUtil - .getObjectsMap(ddpInstance.getParticipantIndexES(), profile.getParticipantGuid(), - ESObjectConstants.WORKFLOWS); - if (Objects.isNull(esMap) || esMap.isEmpty()) return; - removeFamilyMemberWorkflowData(ddpInstance, profile, pData, esMap); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - private void removeFamilyMemberWorkflowData(DDPInstance ddpInstance, ESProfile profile, Map pData, Map esMap) throws IOException { - logger.info("Email in patch data does not match participant profile email, will remove workflows"); - CopyOnWriteArrayList> workflowsList = new CopyOnWriteArrayList<>((List>) esMap.get(ESObjectConstants.WORKFLOWS)); - int startingSize = workflowsList.size(); - workflowsList.forEach(workflow -> { - Map workflowDataMap = (Map) workflow.get(ESObjectConstants.DATA); - String collaboratorParticipantId = workflowDataMap.get(ESObjectConstants.SUBJECT_ID); - if (Objects.isNull(collaboratorParticipantId)) return; - if (collaboratorParticipantId.equals(pData.get(FamilyMemberConstants.COLLABORATOR_PARTICIPANT_ID))) { - workflowsList.remove(workflow); - } - }); - if (startingSize != workflowsList.size()) { - esMap.put(ESObjectConstants.WORKFLOWS, workflowsList); - // Use participant guid here to avoid another ES lookup. - ElasticSearchUtil.updateRequest(profile.getParticipantGuid(), ddpInstance.getParticipantIndexES(), esMap); - } - } - - private void writeFamilyMemberWorklow(Patch patch, DDPInstance ddpInstance, ESProfile profile, Map pData) { - logger.info("Email in patch data matches participant profile email, will update workflows"); - int ddpInstanceIdByGuid = Integer.parseInt(ddpInstance.getDdpInstanceId()); - FieldSettings fieldSettings = new FieldSettings(); - pData.forEach((columnName, columnValue) -> { - if (!fieldSettings.isColumnExportable(ddpInstanceIdByGuid, columnName)) return; - if (!patch.getFieldId().contains(org.broadinstitute.dsm.model.participant.data.ParticipantData.FIELD_TYPE_PARTICIPANTS)) return; - // Use participant guid here to avoid multiple ES lookups. - ElasticSearchUtil.writeWorkflow(WorkflowForES.createInstanceWithStudySpecificData(ddpInstance, - profile.getParticipantGuid(), columnName, columnValue, new WorkflowForES.StudySpecificData( - pData.get(FamilyMemberConstants.COLLABORATOR_PARTICIPANT_ID), - pData.get(FamilyMemberConstants.FIRSTNAME), - pData.get(FamilyMemberConstants.LASTNAME))), false); - }); - } - - private void insertDdpParticipantRecord(int participantId) { - ParticipantRecordDto participantRecordDto = - new ParticipantRecordDto.Builder(participantId, System.currentTimeMillis()) - .withChangedBy("SYSTEM") - .builder(); - new ParticipantRecordDao().create(participantRecordDto); - } - - private int insertDdpParticipant(Patch patch, DDPInstance ddpInstance) { - ParticipantDto participantDto = - new ParticipantDto.Builder(Integer.parseInt(ddpInstance.getDdpInstanceId()), System.currentTimeMillis()) - .withDdpParticipantId(patch.getParentId()) - .withLastVersion(0) - .withLastVersionDate("") - .withChangedBy(patch.getUser()) - .build(); - return new ParticipantDao().create(participantDto); - } - - private void writeDSMRecordsToES(@NonNull Patch patch, DDPInstance ddpInstance) { - NameValue nameValue = patch.getNameValue(); - String name = nameValue.getName().substring(nameValue.getName().lastIndexOf('.') + 1); - String type = getTypeFrom(nameValue); - if (type == null) return; - String value = nameValue.getValue().toString(); - Map nameValueMap = new HashMap<>(); - nameValueMap.put(name, value); - if (isMedicalRecord(name, type)) { - ElasticSearchUtil.writeDsmRecord(ddpInstance, Integer.parseInt(patch.getId()), patch.getDdpParticipantId(), - ESObjectConstants.MEDICAL_RECORDS, ESObjectConstants.MEDICAL_RECORDS_ID, nameValueMap); - } - else if (DBConstants.DDP_ONC_HISTORY_DETAIL_ALIAS.equals(type)) { - // TODO relationship between DDP_ONC_HISTORY_DETAIL_ALIAS and TISSUE_RECORDS_FIELD_NAMES - if (ESObjectConstants.TISSUE_RECORDS_FIELD_NAMES.contains(name)) { - if (PARTICIPANT_ID.equals(patch.getParent())) { - ElasticSearchUtil.writeDsmRecord(ddpInstance, Integer.parseInt(patch.getId()), patch.getDdpParticipantId(), - ESObjectConstants.TISSUE_RECORDS, ESObjectConstants.TISSUE_RECORDS_ID, nameValueMap); - } - } - } - } - - private String getTypeFrom(NameValue nameValue) { - String type = null; - if (nameValue.getName().indexOf('.') != -1) { - type = nameValue.getName().substring(0, nameValue.getName().indexOf('.')); - } - else { - return null; - } - return type; - } - - private boolean isMedicalRecord(String name, String type) { - return DBConstants.DDP_MEDICAL_RECORD_ALIAS.equals(type) && ESObjectConstants.MEDICAL_RECORDS_FIELD_NAMES.contains(name); - } - - private void writeESWorkflow(@NonNull Patch patch, @NonNull NameValue nameValue, @NonNull Value action, DDPInstance ddpInstance, String esParticipantId) { - String status = nameValue.getValue() != null ? String.valueOf(nameValue.getValue()) : null; - if (StringUtils.isBlank(status)) { - return; - } - Map data = gson.fromJson(status, new TypeToken>() { - }.getType()); - if (StringUtils.isNotBlank(action.getValue())) { - if (!patch.getFieldId().contains(FamilyMemberConstants.PARTICIPANTS)) { - ElasticSearchUtil.writeWorkflow(WorkflowForES.createInstance(ddpInstance, esParticipantId, action.getName(), action.getValue()), false); - } - else if (ParticipantUtil.matchesApplicantEmail(data.get(FamilyMemberConstants.COLLABORATOR_PARTICIPANT_ID), - participantDataDao.getParticipantDataByParticipantId(patch.getParentId()))) { - ElasticSearchUtil.writeWorkflow(WorkflowForES.createInstanceWithStudySpecificData(ddpInstance, - esParticipantId, action.getName(), data.get(action.getName()), new WorkflowForES.StudySpecificData( - data.get(FamilyMemberConstants.COLLABORATOR_PARTICIPANT_ID), - data.get(FamilyMemberConstants.FIRSTNAME), - data.get(FamilyMemberConstants.LASTNAME))), false); - } - } - else if (StringUtils.isNotBlank(action.getName()) && data.containsKey(action.getName())) { - if (!patch.getFieldId().contains(FamilyMemberConstants.PARTICIPANTS)) { - ElasticSearchUtil.writeWorkflow(WorkflowForES.createInstance(ddpInstance, esParticipantId, action.getName(), data.get(action.getName())), false); - } - else if (ParticipantUtil.matchesApplicantEmail(data.get(FamilyMemberConstants.COLLABORATOR_PARTICIPANT_ID), - participantDataDao.getParticipantDataByParticipantId(patch.getParentId()))) { - ElasticSearchUtil.writeWorkflow(WorkflowForES.createInstanceWithStudySpecificData(ddpInstance, - esParticipantId, action.getName(), data.get(action.getName()), new WorkflowForES.StudySpecificData( - data.get(FamilyMemberConstants.COLLABORATOR_PARTICIPANT_ID), - data.get(FamilyMemberConstants.FIRSTNAME), - data.get(FamilyMemberConstants.LASTNAME))), false); - } - } - } - - private List setWorkflowRelatedFields(@NonNull Patch patch) { - List nameValues = new ArrayList<>(); - //mr request workflow - if (patch.getNameValue().getName().equals("m.faxSent")) { - nameValues.add(setAdditionalValue("m.faxSentBy", patch, patch.getUser())); - nameValues.add(setAdditionalValue("m.faxConfirmed", patch, patch.getNameValue().getValue())); - } - else if (patch.getNameValue().getName().equals("m.faxSent2")) { - nameValues.add(setAdditionalValue("m.faxSent2By", patch, patch.getUser())); - nameValues.add(setAdditionalValue("m.faxConfirmed2", patch, patch.getNameValue().getValue())); - } - else if (patch.getNameValue().getName().equals("m.faxSent3")) { - nameValues.add(setAdditionalValue("m.faxSent3By", patch, patch.getUser())); - nameValues.add(setAdditionalValue("m.faxConfirmed3", patch, patch.getNameValue().getValue())); - } - //tissue request workflow - else if (patch.getNameValue().getName().equals("oD.tFaxSent")) { - nameValues.add(setAdditionalValue("oD.tFaxSentBy", patch, patch.getUser())); - nameValues.add(setAdditionalValue("oD.tFaxConfirmed", patch, patch.getNameValue().getValue())); - nameValues.add(setAdditionalValue("oD.request", patch, "sent")); - } - else if (patch.getNameValue().getName().equals("oD.tFaxSent2")) { - nameValues.add(setAdditionalValue("oD.tFaxSent2By", patch, patch.getUser())); - nameValues.add(setAdditionalValue("oD.tFaxConfirmed2", patch, patch.getNameValue().getValue())); - nameValues.add(setAdditionalValue("oD.request", patch, "sent")); - } - else if (patch.getNameValue().getName().equals("oD.tFaxSent3")) { - nameValues.add(setAdditionalValue("oD.tFaxSent3By", patch, patch.getUser())); - nameValues.add(setAdditionalValue("oD.tFaxConfirmed3", patch, patch.getNameValue().getValue())); - nameValues.add(setAdditionalValue("oD.request", patch, "sent")); - } - else if (patch.getNameValue().getName().equals("oD.tissueReceived")) { - nameValues.add(setAdditionalValue("oD.request", patch, "received")); - } - else if (patch.getNameValue().getName().equals("t.tissueReturnDate")) { - if (StringUtils.isNotBlank(patch.getNameValue().getValue().toString())) { - nameValues.add(setAdditionalValue("oD.request", new Patch(patch.getParentId(), PARTICIPANT_ID, - null, patch.getUser(), patch.getNameValue(), patch.getNameValues(), patch.getDdpParticipantId()), "returned")); - } - else { - Boolean hasReceivedDate = OncHistoryDetail.hasReceivedDate(patch); - - if (hasReceivedDate) { - nameValues.add(setAdditionalValue("oD.request", new Patch(patch.getParentId(), PARTICIPANT_ID, - null, patch.getUser(), patch.getNameValue(), patch.getNameValues(), patch.getDdpParticipantId()), "received")); - } - else { - nameValues.add(setAdditionalValue("oD.request", new Patch(patch.getParentId(), PARTICIPANT_ID, - null, patch.getUser(), patch.getNameValue(), patch.getNameValues(), patch.getDdpParticipantId()), "sent")); - } - } - } - else if (patch.getNameValue().getName().equals("oD.unableToObtain") && (boolean) patch.getNameValue().getValue()) { - } - else if (patch.getNameValue().getName().equals("oD.unableToObtain") && !(boolean) patch.getNameValue().getValue()) { - Boolean hasReceivedDate = OncHistoryDetail.hasReceivedDate(patch); - - if (hasReceivedDate) { - nameValues.add(setAdditionalValue("oD.request", new Patch(patch.getId(), PARTICIPANT_ID, - patch.getParentId(), patch.getUser(), patch.getNameValue(), patch.getNameValues(), patch.getDdpParticipantId()), "received")); - } - else { - nameValues.add(setAdditionalValue("oD.request", new Patch(patch.getId(), PARTICIPANT_ID, - patch.getParentId(), patch.getUser(), patch.getNameValue(), patch.getNameValues(), patch.getDdpParticipantId()), "sent")); - } - } - return nameValues; - } - - private NameValue setAdditionalValue(String additionalValue, @NonNull Patch patch, @NonNull Object value) { - DBElement dbElement = patchUtil.getColumnNameMap().get(additionalValue); - if (dbElement != null) { - NameValue nameValue = new NameValue(additionalValue, value); - Patch.patch(patch.getId(), patch.getUser(), nameValue, dbElement); - return nameValue; - } - else { - throw new RuntimeException("DBElement not found in ColumnNameMap: " + additionalValue); - } - } - - private void triggerParticipantEvent(DDPInstance ddpInstance, Patch patch, Value action){ - Optional eventType = eventTypeDao.getEventTypeByEventTypeAndInstanceId(action.getName(), ddpInstance.getDdpInstanceId()); - eventType.ifPresent(eventTypeDto -> { - boolean participantHasTriggeredEventByEventType = eventDao.hasTriggeredEventByEventTypeAndDdpParticipantId(action.getName(), patch.getParentId()).orElse(false); - if (!participantHasTriggeredEventByEventType) { - inTransaction((conn) -> { - EventUtil.triggerDDP(conn, eventType, patch.getParentId()); - return null; - }); - } - else { - logger.info("Participant " + patch.getParentId() + " was already triggered for event type " + action.getName()); - } - }); - } } From 3e7db40b56f29567c0ba09bfbec86f52d12021b6 Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 6 Oct 2021 19:39:30 +0400 Subject: [PATCH 019/405] refactor | remove unused exception --- .../java/org/broadinstitute/dsm/model/patch/PatchFactory.java | 3 +++ src/main/java/org/broadinstitute/dsm/route/PatchRoute.java | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java index 478a99b0c..1709df54e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java @@ -25,6 +25,9 @@ public static BasePatch makePatch(Patch patch, NotificationUtil notificationUtil patcher = new ParticipantRecordPatch(patch); } } + if (patcher instanceof NullPatch) { + throw new RuntimeException("Id and parentId was null"); + } return patcher; } diff --git a/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java b/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java index b88cf3152..efa57ff9a 100644 --- a/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java @@ -45,8 +45,7 @@ public Object processRequest(Request request, Response response, String userId) String requestBody = request.body(); Patch patch = GSON.fromJson(requestBody, Patch.class); BasePatch patcher = PatchFactory.makePatch(patch, notificationUtil); - patcher.doPatch(); - throw new RuntimeException("Id and parentId was null"); + return patcher.doPatch(); } catch (DuplicateException e) { return new Result(500, "Duplicate value"); From f43e5bbd22fbafc814b571a1f3d6c30bb805d7a8 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 7 Oct 2021 16:54:33 +0400 Subject: [PATCH 020/405] refactor | minor changes of status codes --- .../java/org/broadinstitute/dsm/route/PatchRoute.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java b/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java index efa57ff9a..643271284 100644 --- a/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java @@ -36,7 +36,8 @@ public PatchRoute(@NonNull NotificationUtil notificationUtil, @NonNull PatchUtil @Override public Object processRequest(Request request, Response response, String userId) throws Exception { if (PatchUtil.getColumnNameMap() == null) { - return new RuntimeException("ColumnNameMap is null!"); + response.status(500); + throw new RuntimeException("ColumnNameMap is null!"); } String userIdRequest = UserUtil.getUserId(request); if (UserUtil.checkUserAccess(null, userId, DBConstants.MR_VIEW, userIdRequest) || UserUtil.checkUserAccess(null, userId, DBConstants.MR_ABSTRACTER, userIdRequest) @@ -48,15 +49,17 @@ public Object processRequest(Request request, Response response, String userId) return patcher.doPatch(); } catch (DuplicateException e) { - return new Result(500, "Duplicate value"); + response.status(500); + throw new RuntimeException("Duplicate value", e); } catch (Exception e) { + response.status(500); throw new RuntimeException("An error occurred while attempting to patch ", e); } } else { - response.status(500); - return new Result(500, UserErrorMessages.NO_RIGHTS); + response.status(403); + return UserErrorMessages.NO_RIGHTS; } } } From 821f19e3e36c6504511fcf643236ecefa6c311be Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 8 Oct 2021 19:16:35 +0400 Subject: [PATCH 021/405] refactor | change names with more meaningful names --- .../{PrimaryKeyPatch.java => ExistingRecordPatch.java} | 6 +++--- .../broadinstitute/dsm/model/patch/PatchFactory.java | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) rename src/main/java/org/broadinstitute/dsm/model/patch/{PrimaryKeyPatch.java => ExistingRecordPatch.java} (98%) diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java similarity index 98% rename from src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java rename to src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java index da558ae6f..3dd57bb2c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/PrimaryKeyPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java @@ -25,13 +25,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class PrimaryKeyPatch extends BasePatch { +public class ExistingRecordPatch extends BasePatch { - private static final Logger logger = LoggerFactory.getLogger(PrimaryKeyPatch.class); + private static final Logger logger = LoggerFactory.getLogger(ExistingRecordPatch.class); private final NotificationUtil notificationUtil; - public PrimaryKeyPatch(Patch patch, NotificationUtil notificationUtil) { + public ExistingRecordPatch(Patch patch, NotificationUtil notificationUtil) { super(patch); this.notificationUtil = notificationUtil; } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java index 1709df54e..b678de469 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java @@ -8,9 +8,9 @@ public class PatchFactory { public static BasePatch makePatch(Patch patch, NotificationUtil notificationUtil) { BasePatch patcher = new NullPatch(); - if (hasPrimaryKey(patch)) { - patcher = new PrimaryKeyPatch(patch, notificationUtil); - } else if (isParentWithPrimaryKey(patch)) { + if (isExistingRecord(patch)) { + patcher = new ExistingRecordPatch(patch, notificationUtil); + } else if (isParentWithExistingKey(patch)) { if (isParentParticipantId(patch)) { if (isMedicalRecordAbstractionFieldId(patch)) { patcher = new AbstractionPatch(patch); @@ -31,11 +31,11 @@ public static BasePatch makePatch(Patch patch, NotificationUtil notificationUtil return patcher; } - private static boolean hasPrimaryKey(Patch patch) { + private static boolean isExistingRecord(Patch patch) { return StringUtils.isNotBlank(patch.getId()); } - private static boolean isParentWithPrimaryKey(Patch patch) { + private static boolean isParentWithExistingKey(Patch patch) { return StringUtils.isNotBlank(patch.getParent()) && StringUtils.isNotBlank(patch.getParentId()); } From a79f22d890bba7cbe5288be52cbd44abf62360b8 Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 11 Oct 2021 14:48:14 +0400 Subject: [PATCH 022/405] refactor | tweak code based on suggestions and comments --- src/main/java/org/broadinstitute/dsm/model/Patch.java | 2 +- .../dsm/model/patch/AbstractionPatch.java | 4 +--- .../dsm/model/patch/ExistingRecordPatch.java | 8 ++------ ...icalRecordPatch.java => OncHistoryDetailPatch.java} | 10 ++++------ .../broadinstitute/dsm/model/patch/PatchFactory.java | 2 +- 5 files changed, 9 insertions(+), 17 deletions(-) rename src/main/java/org/broadinstitute/dsm/model/patch/{MedicalRecordPatch.java => OncHistoryDetailPatch.java} (89%) diff --git a/src/main/java/org/broadinstitute/dsm/model/Patch.java b/src/main/java/org/broadinstitute/dsm/model/Patch.java index e4e740a82..f18023ae5 100644 --- a/src/main/java/org/broadinstitute/dsm/model/Patch.java +++ b/src/main/java/org/broadinstitute/dsm/model/Patch.java @@ -151,7 +151,7 @@ else if (StringUtils.isBlank(String.valueOf(nameValue.getValue()))) { return true; } - public static Boolean isValueUnique(@NonNull DBElement dbElement){ + public static Boolean isValueUnique(@NonNull DBElement dbElement){ SimpleResult results = inTransaction((conn) -> { SimpleResult dbVals = new SimpleResult(); try (PreparedStatement stmt = conn.prepareStatement(SQL_CHECK_UNIQUE. diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java index 068c47fb5..234e9d8c7 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java @@ -53,9 +53,7 @@ Optional processEachNameValue(NameValue nameValue, DBElement dbElement) if (StringUtils.isBlank(primaryKeyId)) { primaryKeyId = AbstractionWrapper.createNewAbstractionFieldValue(patch.getParentId(), patch.getFieldId(), patch.getUser(), nameValue, dbElement); } - if (!Patch.patch(primaryKeyId, patch.getUser(), nameValue, dbElement)) { - throw new RuntimeException("An error occurred while attempting to patch "); - } + Patch.patch(primaryKeyId, patch.getUser(), nameValue, dbElement); return Optional.ofNullable(primaryKeyId); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java index 3dd57bb2c..3de48a960 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java @@ -60,9 +60,7 @@ public Object patchNameValuePair() { @Override Optional processEachNameValue(NameValue nameValue, DBElement dbElement) { Optional maybeUpdatedNameValue = Optional.empty(); - if (!Patch.patch(patch.getId(), patch.getUser(), nameValue, dbElement)) { - throw new RuntimeException("An error occurred while attempting to patch "); - } + Patch.patch(patch.getId(), patch.getUser(), nameValue, dbElement); if (hasQuestion(nameValue)) { maybeUpdatedNameValue = sendNotificationEmailAndUpdateStatus(patch, nameValue, dbElement); } @@ -94,9 +92,7 @@ private Optional sendNotificationEmailAndUpdateStatus(Patch patch, NameV jsonObject.put("questions", questionArray); String str = jsonObject.toString(); nameValue.setValue(str); - if (!Patch.patch(patch.getId(), patch.getUser(), nameValue, dbElement)) { - throw new RuntimeException("An error occurred while attempting to patch "); - } + Patch.patch(patch.getId(), patch.getUser(), nameValue, dbElement); maybeUpdatedNameValue = Optional.of(nameValue); } return maybeUpdatedNameValue; diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/MedicalRecordPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/OncHistoryDetailPatch.java similarity index 89% rename from src/main/java/org/broadinstitute/dsm/model/patch/MedicalRecordPatch.java rename to src/main/java/org/broadinstitute/dsm/model/patch/OncHistoryDetailPatch.java index e40a826fa..add075421 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/MedicalRecordPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/OncHistoryDetailPatch.java @@ -13,17 +13,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class MedicalRecordPatch extends BasePatch { +public class OncHistoryDetailPatch extends BasePatch { private static final String ONC_HISTORY_DETAIL_ID = "oncHistoryDetailId"; private Number mrID; private String oncHistoryDetailId; - static final Logger logger = LoggerFactory.getLogger(MedicalRecordPatch.class); + static final Logger logger = LoggerFactory.getLogger(OncHistoryDetailPatch.class); - public MedicalRecordPatch(Patch patch) { + public OncHistoryDetailPatch(Patch patch) { super(patch); } @@ -91,9 +91,7 @@ Object handleSingleNameValue(DBElement dbElement) { @Override Optional processEachNameValue(NameValue nameValue, DBElement dbElement) { - if (!Patch.patch(oncHistoryDetailId, patch.getUser(), nameValue, dbElement)) { - throw new RuntimeException("An error occurred while attempting to patch "); - } + Patch.patch(oncHistoryDetailId, patch.getUser(), nameValue, dbElement); return Optional.empty(); } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java index b678de469..39ea00a2c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java @@ -15,7 +15,7 @@ public static BasePatch makePatch(Patch patch, NotificationUtil notificationUtil if (isMedicalRecordAbstractionFieldId(patch)) { patcher = new AbstractionPatch(patch); } else { - patcher = new MedicalRecordPatch(patch); + patcher = new OncHistoryDetailPatch(patch); } } else if (isTissueRelatedOncHistoryId(patch)) { patcher = new TissuePatch(patch); From cb310d2a5f8ac0763d956d5ef4f9373ff7fc7fa7 Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 11 Oct 2021 15:26:36 +0400 Subject: [PATCH 023/405] refactor | move Patch.java to src/main/java/org/broadinstitute/dsm/model/patch --- src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java | 2 +- .../java/org/broadinstitute/dsm/model/AbstractionWrapper.java | 1 + .../org/broadinstitute/dsm/model/patch/AbstractionPatch.java | 1 - .../java/org/broadinstitute/dsm/model/patch/BasePatch.java | 1 - .../broadinstitute/dsm/model/patch/ExistingRecordPatch.java | 1 - .../broadinstitute/dsm/model/patch/OncHistoryDetailPatch.java | 1 - .../broadinstitute/dsm/model/patch/ParticipantDataPatch.java | 1 - .../dsm/model/patch/ParticipantRecordPatch.java | 1 - .../java/org/broadinstitute/dsm/model/{ => patch}/Patch.java | 4 +++- .../java/org/broadinstitute/dsm/model/patch/PatchFactory.java | 1 - .../java/org/broadinstitute/dsm/model/patch/TissuePatch.java | 1 - .../java/org/broadinstitute/dsm/route/AbstractionRoute.java | 1 + src/main/java/org/broadinstitute/dsm/route/PatchRoute.java | 3 +-- .../java/org/broadinstitute/dsm/util/AbstractionUtil.java | 2 +- 14 files changed, 8 insertions(+), 13 deletions(-) rename src/main/java/org/broadinstitute/dsm/model/{ => patch}/Patch.java (98%) diff --git a/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java b/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java index c556e19f0..19430bf46 100644 --- a/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java +++ b/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java @@ -8,7 +8,7 @@ import org.broadinstitute.dsm.db.structure.DbDateConversion; import org.broadinstitute.dsm.db.structure.SqlDateConverter; import org.broadinstitute.dsm.db.structure.TableName; -import org.broadinstitute.dsm.model.Patch; +import org.broadinstitute.dsm.model.patch.Patch; import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.statics.QueryExtension; import org.broadinstitute.dsm.util.DBUtil; diff --git a/src/main/java/org/broadinstitute/dsm/model/AbstractionWrapper.java b/src/main/java/org/broadinstitute/dsm/model/AbstractionWrapper.java index 390111ccc..16c1e190e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/AbstractionWrapper.java +++ b/src/main/java/org/broadinstitute/dsm/model/AbstractionWrapper.java @@ -8,6 +8,7 @@ import org.broadinstitute.dsm.db.AbstractionFinal; import org.broadinstitute.dsm.db.AbstractionGroup; import org.broadinstitute.dsm.db.structure.DBElement; +import org.broadinstitute.dsm.model.patch.Patch; import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.util.AbstractionUtil; import org.slf4j.Logger; diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java index 234e9d8c7..31deda50d 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java @@ -3,7 +3,6 @@ import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.AbstractionWrapper; import org.broadinstitute.dsm.model.NameValue; -import org.broadinstitute.dsm.model.Patch; import spark.utils.StringUtils; import java.util.HashMap; diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index 45e782adb..da7100885 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -22,7 +22,6 @@ import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.export.WorkflowForES; import org.broadinstitute.dsm.model.NameValue; -import org.broadinstitute.dsm.model.Patch; import org.broadinstitute.dsm.model.Value; import org.broadinstitute.dsm.model.elasticsearch.ESProfile; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java index 3de48a960..e1c2c11c8 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java @@ -11,7 +11,6 @@ import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.export.WorkflowForES; import org.broadinstitute.dsm.model.NameValue; -import org.broadinstitute.dsm.model.Patch; import org.broadinstitute.dsm.model.Value; import org.broadinstitute.dsm.model.elasticsearch.ESProfile; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/OncHistoryDetailPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/OncHistoryDetailPatch.java index add075421..e542c3516 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/OncHistoryDetailPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/OncHistoryDetailPatch.java @@ -7,7 +7,6 @@ import org.broadinstitute.dsm.db.OncHistoryDetail; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; -import org.broadinstitute.dsm.model.Patch; import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.util.MedicalRecordUtil; import org.slf4j.Logger; diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantDataPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantDataPatch.java index 00552dc69..a41474b0b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantDataPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantDataPatch.java @@ -4,7 +4,6 @@ import org.broadinstitute.dsm.db.dao.settings.EventTypeDao; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; -import org.broadinstitute.dsm.model.Patch; import org.broadinstitute.dsm.model.Value; import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantRecordPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantRecordPatch.java index 2e5a83ad3..84192d422 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantRecordPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantRecordPatch.java @@ -9,7 +9,6 @@ import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantRecordDto; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; -import org.broadinstitute.dsm.model.Patch; public class ParticipantRecordPatch extends BasePatch { diff --git a/src/main/java/org/broadinstitute/dsm/model/Patch.java b/src/main/java/org/broadinstitute/dsm/model/patch/Patch.java similarity index 98% rename from src/main/java/org/broadinstitute/dsm/model/Patch.java rename to src/main/java/org/broadinstitute/dsm/model/patch/Patch.java index f18023ae5..4f9b2c201 100644 --- a/src/main/java/org/broadinstitute/dsm/model/Patch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/Patch.java @@ -1,4 +1,4 @@ -package org.broadinstitute.dsm.model; +package org.broadinstitute.dsm.model.patch; import com.google.gson.Gson; import lombok.Data; @@ -7,6 +7,8 @@ import org.broadinstitute.ddp.db.SimpleResult; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.exception.DuplicateException; +import org.broadinstitute.dsm.model.NameValue; +import org.broadinstitute.dsm.model.Value; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java index 39ea00a2c..bc34f14a8 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java @@ -1,7 +1,6 @@ package org.broadinstitute.dsm.model.patch; import org.apache.commons.lang3.StringUtils; -import org.broadinstitute.dsm.model.Patch; import org.broadinstitute.dsm.util.NotificationUtil; public class PatchFactory { diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java index b40d37024..a6e2fe931 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java @@ -5,7 +5,6 @@ import org.broadinstitute.dsm.db.Tissue; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; -import org.broadinstitute.dsm.model.Patch; public class TissuePatch extends BasePatch { diff --git a/src/main/java/org/broadinstitute/dsm/route/AbstractionRoute.java b/src/main/java/org/broadinstitute/dsm/route/AbstractionRoute.java index 44f6d2282..46ca0f664 100644 --- a/src/main/java/org/broadinstitute/dsm/route/AbstractionRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/AbstractionRoute.java @@ -7,6 +7,7 @@ import org.broadinstitute.dsm.db.*; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.*; +import org.broadinstitute.dsm.model.patch.Patch; import org.broadinstitute.dsm.security.RequestHandler; import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.statics.RequestParameter; diff --git a/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java b/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java index 643271284..165943d11 100644 --- a/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/PatchRoute.java @@ -3,9 +3,8 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import lombok.NonNull; -import org.broadinstitute.ddp.handlers.util.Result; import org.broadinstitute.dsm.exception.DuplicateException; -import org.broadinstitute.dsm.model.Patch; +import org.broadinstitute.dsm.model.patch.Patch; import org.broadinstitute.dsm.model.patch.BasePatch; import org.broadinstitute.dsm.model.patch.PatchFactory; import org.broadinstitute.dsm.security.RequestHandler; diff --git a/src/main/java/org/broadinstitute/dsm/util/AbstractionUtil.java b/src/main/java/org/broadinstitute/dsm/util/AbstractionUtil.java index c827386e1..3bfadd00f 100644 --- a/src/main/java/org/broadinstitute/dsm/util/AbstractionUtil.java +++ b/src/main/java/org/broadinstitute/dsm/util/AbstractionUtil.java @@ -8,7 +8,7 @@ import org.broadinstitute.dsm.db.AbstractionFieldValue; import org.broadinstitute.dsm.db.AbstractionGroup; import org.broadinstitute.dsm.model.AbstractionQCWrapper; -import org.broadinstitute.dsm.model.Patch; +import org.broadinstitute.dsm.model.patch.Patch; import org.broadinstitute.dsm.model.Value; import org.broadinstitute.dsm.statics.DBConstants; import org.json.JSONArray; From bb7e0d39d20f75692f7102a56e49d3e95f379050 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 14 Oct 2021 11:07:36 +0400 Subject: [PATCH 024/405] DDP-7047 | start of implementing elastic exporter --- .../broadinstitute/dsm/db/dao/kit/BSPDummyKitDao.java | 4 ++-- .../dsm/export/WorkflowAndFamilyIdExporter.java | 2 +- .../org/broadinstitute/dsm/model/at/DefaultValues.java | 6 ++---- .../model/{elasticsearch => elastic}/ESActivities.java | 2 +- .../model/{elasticsearch => elastic}/ESAddress.java | 2 +- .../model/{elasticsearch => elastic}/ESComputed.java | 2 +- .../dsm/model/{elasticsearch => elastic}/ESDsm.java | 2 +- .../model/{elasticsearch => elastic}/ESProfile.java | 2 +- .../dsm/model/elastic/export/Exportable.java | 9 +++++++++ .../search}/ElasticSearch.java | 2 +- .../search}/ElasticSearchParticipantDto.java | 7 ++++++- .../search}/ElasticSearchable.java | 2 +- .../filter/participant/BaseFilterParticipantList.java | 3 +-- .../filter/participant/EmptyFilterParticipantList.java | 2 +- .../dsm/model/participant/ParticipantWrapper.java | 8 ++++---- .../dsm/model/participant/ParticipantWrapperDto.java | 2 +- .../dsm/model/participant/data/ParticipantData.java | 2 +- .../org/broadinstitute/dsm/model/patch/BasePatch.java | 2 +- .../dsm/model/patch/ExistingRecordPatch.java | 2 +- .../dsm/model/rgp/AutomaticProbandDataCreator.java | 4 ++-- .../broadinstitute/dsm/route/ClinicalKitsRoute.java | 8 ++++---- .../dsm/route/CreateClinicalDummyKitRoute.java | 4 ++-- .../org/broadinstitute/dsm/route/DashboardRoute.java | 2 +- .../org/broadinstitute/dsm/route/KitUploadRoute.java | 6 +++--- .../dsm/route/participant/GetParticipantRoute.java | 4 +--- .../org/broadinstitute/dsm/util/ElasticSearchUtil.java | 8 ++++---- .../org/broadinstitute/dsm/util/ParticipantUtil.java | 5 ++--- .../java/org/broadinstitute/dsm/ElasticSearchTest.java | 5 ++--- .../search}/ElasticSearchTest.java | 7 +++---- .../dsm/model/participant/ParticipantWrapperTest.java | 10 ++++------ .../broadinstitute/dsm/route/KitUploadRouteTest.java | 5 ++--- 31 files changed, 67 insertions(+), 64 deletions(-) rename src/main/java/org/broadinstitute/dsm/model/{elasticsearch => elastic}/ESActivities.java (94%) rename src/main/java/org/broadinstitute/dsm/model/{elasticsearch => elastic}/ESAddress.java (95%) rename src/main/java/org/broadinstitute/dsm/model/{elasticsearch => elastic}/ESComputed.java (80%) rename src/main/java/org/broadinstitute/dsm/model/{elasticsearch => elastic}/ESDsm.java (94%) rename src/main/java/org/broadinstitute/dsm/model/{elasticsearch => elastic}/ESProfile.java (95%) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java rename src/main/java/org/broadinstitute/dsm/model/{elasticsearch => elastic/search}/ElasticSearch.java (99%) rename src/main/java/org/broadinstitute/dsm/model/{elasticsearch => elastic/search}/ElasticSearchParticipantDto.java (93%) rename src/main/java/org/broadinstitute/dsm/model/{elasticsearch => elastic/search}/ElasticSearchable.java (93%) rename src/test/java/org/broadinstitute/dsm/model/{elasticsearch => elastic/search}/ElasticSearchTest.java (95%) diff --git a/src/main/java/org/broadinstitute/dsm/db/dao/kit/BSPDummyKitDao.java b/src/main/java/org/broadinstitute/dsm/db/dao/kit/BSPDummyKitDao.java index 0e9e8376a..712789c69 100644 --- a/src/main/java/org/broadinstitute/dsm/db/dao/kit/BSPDummyKitDao.java +++ b/src/main/java/org/broadinstitute/dsm/db/dao/kit/BSPDummyKitDao.java @@ -4,8 +4,8 @@ import org.broadinstitute.dsm.db.DDPInstance; import org.broadinstitute.dsm.db.dao.Dao; import org.broadinstitute.dsm.db.dto.kit.ClinicalKitDto; -import org.broadinstitute.dsm.model.elasticsearch.ESProfile; -import org.broadinstitute.dsm.model.elasticsearch.ElasticSearchParticipantDto; +import org.broadinstitute.dsm.model.elastic.ESProfile; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.slf4j.Logger; diff --git a/src/main/java/org/broadinstitute/dsm/export/WorkflowAndFamilyIdExporter.java b/src/main/java/org/broadinstitute/dsm/export/WorkflowAndFamilyIdExporter.java index b34e37d82..cd7b68164 100644 --- a/src/main/java/org/broadinstitute/dsm/export/WorkflowAndFamilyIdExporter.java +++ b/src/main/java/org/broadinstitute/dsm/export/WorkflowAndFamilyIdExporter.java @@ -8,7 +8,7 @@ import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; import org.broadinstitute.dsm.model.Value; -import org.broadinstitute.dsm.model.elasticsearch.ESProfile; +import org.broadinstitute.dsm.model.elastic.ESProfile; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; diff --git a/src/main/java/org/broadinstitute/dsm/model/at/DefaultValues.java b/src/main/java/org/broadinstitute/dsm/model/at/DefaultValues.java index ac59ef00f..2719250e6 100644 --- a/src/main/java/org/broadinstitute/dsm/model/at/DefaultValues.java +++ b/src/main/java/org/broadinstitute/dsm/model/at/DefaultValues.java @@ -12,12 +12,10 @@ import org.broadinstitute.dsm.db.dto.bookmark.BookmarkDto; import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; -import org.broadinstitute.dsm.model.elasticsearch.ESProfile; -import org.broadinstitute.dsm.model.elasticsearch.ElasticSearchParticipantDto; +import org.broadinstitute.dsm.model.elastic.ESProfile; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.settings.field.FieldSettings; import org.broadinstitute.dsm.model.participant.data.ParticipantData; -import org.broadinstitute.dsm.statics.ESObjectConstants; -import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/org/broadinstitute/dsm/model/elasticsearch/ESActivities.java b/src/main/java/org/broadinstitute/dsm/model/elastic/ESActivities.java similarity index 94% rename from src/main/java/org/broadinstitute/dsm/model/elasticsearch/ESActivities.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/ESActivities.java index cc9f79505..6e7689ccd 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elasticsearch/ESActivities.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/ESActivities.java @@ -1,4 +1,4 @@ -package org.broadinstitute.dsm.model.elasticsearch; +package org.broadinstitute.dsm.model.elastic; import java.util.List; import java.util.Map; diff --git a/src/main/java/org/broadinstitute/dsm/model/elasticsearch/ESAddress.java b/src/main/java/org/broadinstitute/dsm/model/elastic/ESAddress.java similarity index 95% rename from src/main/java/org/broadinstitute/dsm/model/elasticsearch/ESAddress.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/ESAddress.java index 99c5e87f3..687f90b11 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elasticsearch/ESAddress.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/ESAddress.java @@ -1,4 +1,4 @@ -package org.broadinstitute.dsm.model.elasticsearch; +package org.broadinstitute.dsm.model.elastic; import com.google.gson.annotations.SerializedName; import lombok.Setter; diff --git a/src/main/java/org/broadinstitute/dsm/model/elasticsearch/ESComputed.java b/src/main/java/org/broadinstitute/dsm/model/elastic/ESComputed.java similarity index 80% rename from src/main/java/org/broadinstitute/dsm/model/elasticsearch/ESComputed.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/ESComputed.java index 5205b79b1..faab2b285 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elasticsearch/ESComputed.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/ESComputed.java @@ -1,4 +1,4 @@ -package org.broadinstitute.dsm.model.elasticsearch; +package org.broadinstitute.dsm.model.elastic; import com.google.gson.annotations.SerializedName; import lombok.Getter; diff --git a/src/main/java/org/broadinstitute/dsm/model/elasticsearch/ESDsm.java b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java similarity index 94% rename from src/main/java/org/broadinstitute/dsm/model/elasticsearch/ESDsm.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java index a2ab53a4f..8967c3710 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elasticsearch/ESDsm.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java @@ -1,4 +1,4 @@ -package org.broadinstitute.dsm.model.elasticsearch; +package org.broadinstitute.dsm.model.elastic; import java.util.List; import java.util.Map; diff --git a/src/main/java/org/broadinstitute/dsm/model/elasticsearch/ESProfile.java b/src/main/java/org/broadinstitute/dsm/model/elastic/ESProfile.java similarity index 95% rename from src/main/java/org/broadinstitute/dsm/model/elasticsearch/ESProfile.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/ESProfile.java index 84b824e3b..4763b3fd3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elasticsearch/ESProfile.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/ESProfile.java @@ -1,4 +1,4 @@ -package org.broadinstitute.dsm.model.elasticsearch; +package org.broadinstitute.dsm.model.elastic; import com.google.gson.annotations.SerializedName; import lombok.Data; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java new file mode 100644 index 000000000..c773311ca --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java @@ -0,0 +1,9 @@ +package org.broadinstitute.dsm.model.elastic.export; + +import java.util.Map; + +public interface Exportable { + + void export(Map data); + +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elasticsearch/ElasticSearch.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java similarity index 99% rename from src/main/java/org/broadinstitute/dsm/model/elasticsearch/ElasticSearch.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java index fb9ce8405..14f960dde 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elasticsearch/ElasticSearch.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java @@ -1,4 +1,4 @@ -package org.broadinstitute.dsm.model.elasticsearch; +package org.broadinstitute.dsm.model.elastic.search; import java.io.IOException; import java.util.ArrayList; diff --git a/src/main/java/org/broadinstitute/dsm/model/elasticsearch/ElasticSearchParticipantDto.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchParticipantDto.java similarity index 93% rename from src/main/java/org/broadinstitute/dsm/model/elasticsearch/ElasticSearchParticipantDto.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchParticipantDto.java index 18618b348..261fb9e38 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elasticsearch/ElasticSearchParticipantDto.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchParticipantDto.java @@ -1,4 +1,4 @@ -package org.broadinstitute.dsm.model.elasticsearch; +package org.broadinstitute.dsm.model.elastic.search; import java.util.List; import java.util.Map; @@ -6,6 +6,11 @@ import lombok.Setter; import org.apache.commons.lang3.StringUtils; +import org.broadinstitute.dsm.model.elastic.ESActivities; +import org.broadinstitute.dsm.model.elastic.ESAddress; +import org.broadinstitute.dsm.model.elastic.ESComputed; +import org.broadinstitute.dsm.model.elastic.ESDsm; +import org.broadinstitute.dsm.model.elastic.ESProfile; @Setter public class ElasticSearchParticipantDto { diff --git a/src/main/java/org/broadinstitute/dsm/model/elasticsearch/ElasticSearchable.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchable.java similarity index 93% rename from src/main/java/org/broadinstitute/dsm/model/elasticsearch/ElasticSearchable.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchable.java index c645b0915..4aef55310 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elasticsearch/ElasticSearchable.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchable.java @@ -1,4 +1,4 @@ -package org.broadinstitute.dsm.model.elasticsearch; +package org.broadinstitute.dsm.model.elastic.search; import java.util.List; diff --git a/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java b/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java index ba5c6110d..803211b1f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java +++ b/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java @@ -18,11 +18,10 @@ import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.Filter; -import org.broadinstitute.dsm.model.elasticsearch.ElasticSearch; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; import org.broadinstitute.dsm.model.participant.ParticipantWrapper; import org.broadinstitute.dsm.model.filter.BaseFilter; import org.broadinstitute.dsm.model.filter.Filterable; -import org.broadinstitute.dsm.model.participant.ParticipantWrapperDto; import org.broadinstitute.dsm.model.participant.ParticipantWrapperPayload; import org.broadinstitute.dsm.model.participant.ParticipantWrapperResult; import org.broadinstitute.dsm.statics.DBConstants; diff --git a/src/main/java/org/broadinstitute/dsm/model/filter/participant/EmptyFilterParticipantList.java b/src/main/java/org/broadinstitute/dsm/model/filter/participant/EmptyFilterParticipantList.java index b7dd6fd69..1af0e4bdb 100644 --- a/src/main/java/org/broadinstitute/dsm/model/filter/participant/EmptyFilterParticipantList.java +++ b/src/main/java/org/broadinstitute/dsm/model/filter/participant/EmptyFilterParticipantList.java @@ -4,7 +4,7 @@ import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; -import org.broadinstitute.dsm.model.elasticsearch.ElasticSearch; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; import org.broadinstitute.dsm.model.participant.ParticipantWrapper; import org.broadinstitute.dsm.model.participant.ParticipantWrapperPayload; import org.broadinstitute.dsm.model.participant.ParticipantWrapperResult; diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java index 0b5b3d425..f55b19ed4 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java @@ -9,10 +9,10 @@ import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; -import org.broadinstitute.dsm.model.elasticsearch.ESProfile; -import org.broadinstitute.dsm.model.elasticsearch.ElasticSearch; -import org.broadinstitute.dsm.model.elasticsearch.ElasticSearchParticipantDto; -import org.broadinstitute.dsm.model.elasticsearch.ElasticSearchable; +import org.broadinstitute.dsm.model.elastic.ESProfile; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearchable; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import org.broadinstitute.dsm.model.at.DefaultValues; import org.broadinstitute.dsm.statics.DBConstants; diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapperDto.java b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapperDto.java index c56d40fc1..1050e1469 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapperDto.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapperDto.java @@ -17,7 +17,7 @@ import org.broadinstitute.dsm.db.OncHistoryDetail; import org.broadinstitute.dsm.db.Participant; import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; -import org.broadinstitute.dsm.model.elasticsearch.ElasticSearchParticipantDto; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java b/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java index 7b123a187..4ec0056da 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java @@ -17,7 +17,7 @@ import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; -import org.broadinstitute.dsm.model.elasticsearch.ESProfile; +import org.broadinstitute.dsm.model.elastic.ESProfile; import org.broadinstitute.dsm.util.ParticipantUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index da7100885..ce9dbf4d7 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -23,7 +23,7 @@ import org.broadinstitute.dsm.export.WorkflowForES; import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.Value; -import org.broadinstitute.dsm.model.elasticsearch.ESProfile; +import org.broadinstitute.dsm.model.elastic.ESProfile; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java index e1c2c11c8..96d58c0aa 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java @@ -12,7 +12,7 @@ import org.broadinstitute.dsm.export.WorkflowForES; import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.Value; -import org.broadinstitute.dsm.model.elasticsearch.ESProfile; +import org.broadinstitute.dsm.model.elastic.ESProfile; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import org.broadinstitute.dsm.model.settings.field.FieldSettings; import org.broadinstitute.dsm.statics.DBConstants; diff --git a/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java b/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java index b2f37f126..53e6298be 100644 --- a/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java +++ b/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java @@ -16,8 +16,8 @@ import org.broadinstitute.dsm.export.WorkflowForES; import org.broadinstitute.dsm.model.ddp.DDPActivityConstants; import org.broadinstitute.dsm.model.defaultvalues.Defaultable; -import org.broadinstitute.dsm.model.elasticsearch.ESActivities; -import org.broadinstitute.dsm.model.elasticsearch.ElasticSearchParticipantDto; +import org.broadinstitute.dsm.model.elastic.ESActivities; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import org.broadinstitute.dsm.model.participant.data.FamilyMemberDetails; import org.broadinstitute.dsm.model.participant.data.ParticipantData; diff --git a/src/main/java/org/broadinstitute/dsm/route/ClinicalKitsRoute.java b/src/main/java/org/broadinstitute/dsm/route/ClinicalKitsRoute.java index 89977173d..e73456495 100644 --- a/src/main/java/org/broadinstitute/dsm/route/ClinicalKitsRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/ClinicalKitsRoute.java @@ -12,10 +12,10 @@ import org.broadinstitute.dsm.model.BSPKit; import org.broadinstitute.dsm.model.bsp.BSPKitInfo; import org.broadinstitute.dsm.model.bsp.BSPKitStatus; -import org.broadinstitute.dsm.model.elasticsearch.ESDsm; -import org.broadinstitute.dsm.model.elasticsearch.ESProfile; -import org.broadinstitute.dsm.model.elasticsearch.ElasticSearch; -import org.broadinstitute.dsm.model.elasticsearch.ElasticSearchParticipantDto; +import org.broadinstitute.dsm.model.elastic.ESDsm; +import org.broadinstitute.dsm.model.elastic.ESProfile; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.statics.RequestParameter; import org.broadinstitute.dsm.util.NotificationUtil; import org.slf4j.Logger; diff --git a/src/main/java/org/broadinstitute/dsm/route/CreateClinicalDummyKitRoute.java b/src/main/java/org/broadinstitute/dsm/route/CreateClinicalDummyKitRoute.java index 003ddf8bf..8eda32f2e 100644 --- a/src/main/java/org/broadinstitute/dsm/route/CreateClinicalDummyKitRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/CreateClinicalDummyKitRoute.java @@ -6,8 +6,8 @@ import org.broadinstitute.dsm.db.KitType; import org.broadinstitute.dsm.db.dao.bookmark.BookmarkDao; import org.broadinstitute.dsm.db.dao.kit.BSPDummyKitDao; -import org.broadinstitute.dsm.model.elasticsearch.ESProfile; -import org.broadinstitute.dsm.model.elasticsearch.ElasticSearchParticipantDto; +import org.broadinstitute.dsm.model.elastic.ESProfile; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.statics.RequestParameter; import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.slf4j.Logger; diff --git a/src/main/java/org/broadinstitute/dsm/route/DashboardRoute.java b/src/main/java/org/broadinstitute/dsm/route/DashboardRoute.java index 3f8d431ed..37524ef71 100644 --- a/src/main/java/org/broadinstitute/dsm/route/DashboardRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/DashboardRoute.java @@ -9,7 +9,7 @@ import org.broadinstitute.dsm.db.KitType; import org.broadinstitute.dsm.db.*; import org.broadinstitute.dsm.model.*; -import org.broadinstitute.dsm.model.elasticsearch.ElasticSearchParticipantDto; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.participant.ParticipantWrapperDto; import org.broadinstitute.dsm.security.RequestHandler; import org.broadinstitute.dsm.statics.*; diff --git a/src/main/java/org/broadinstitute/dsm/route/KitUploadRoute.java b/src/main/java/org/broadinstitute/dsm/route/KitUploadRoute.java index f2acb2bc5..5a25c7826 100644 --- a/src/main/java/org/broadinstitute/dsm/route/KitUploadRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/KitUploadRoute.java @@ -18,9 +18,9 @@ import org.broadinstitute.dsm.exception.FileWrongSeparator; import org.broadinstitute.dsm.exception.UploadLineException; import org.broadinstitute.dsm.model.*; -import org.broadinstitute.dsm.model.elasticsearch.ESProfile; -import org.broadinstitute.dsm.model.elasticsearch.ElasticSearch; -import org.broadinstitute.dsm.model.elasticsearch.ElasticSearchParticipantDto; +import org.broadinstitute.dsm.model.elastic.ESProfile; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.security.RequestHandler; import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.statics.RoutePath; diff --git a/src/main/java/org/broadinstitute/dsm/route/participant/GetParticipantRoute.java b/src/main/java/org/broadinstitute/dsm/route/participant/GetParticipantRoute.java index adbc74234..a9decff90 100644 --- a/src/main/java/org/broadinstitute/dsm/route/participant/GetParticipantRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/participant/GetParticipantRoute.java @@ -3,13 +3,11 @@ import java.util.Map; import org.apache.commons.lang3.StringUtils; -import org.broadinstitute.dsm.db.DDPInstance; import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; -import org.broadinstitute.dsm.model.elasticsearch.ElasticSearch; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; import org.broadinstitute.dsm.model.participant.ParticipantWrapper; import org.broadinstitute.dsm.model.participant.ParticipantWrapperPayload; -import org.broadinstitute.dsm.model.participant.ParticipantWrapperResult; import org.broadinstitute.dsm.security.RequestHandler; import org.broadinstitute.dsm.statics.RoutePath; import org.broadinstitute.dsm.statics.UserErrorMessages; diff --git a/src/main/java/org/broadinstitute/dsm/util/ElasticSearchUtil.java b/src/main/java/org/broadinstitute/dsm/util/ElasticSearchUtil.java index ee2798cc8..cabf6c2e4 100644 --- a/src/main/java/org/broadinstitute/dsm/util/ElasticSearchUtil.java +++ b/src/main/java/org/broadinstitute/dsm/util/ElasticSearchUtil.java @@ -19,10 +19,10 @@ import org.broadinstitute.dsm.export.WorkflowForES; import org.broadinstitute.dsm.model.Filter; import org.broadinstitute.dsm.model.ddp.DDPParticipant; -import org.broadinstitute.dsm.model.elasticsearch.ESAddress; -import org.broadinstitute.dsm.model.elasticsearch.ESProfile; -import org.broadinstitute.dsm.model.elasticsearch.ElasticSearch; -import org.broadinstitute.dsm.model.elasticsearch.ElasticSearchParticipantDto; +import org.broadinstitute.dsm.model.elastic.ESAddress; +import org.broadinstitute.dsm.model.elastic.ESProfile; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.gbf.Address; import org.broadinstitute.dsm.statics.ApplicationConfigConstants; import org.broadinstitute.dsm.statics.DBConstants; diff --git a/src/main/java/org/broadinstitute/dsm/util/ParticipantUtil.java b/src/main/java/org/broadinstitute/dsm/util/ParticipantUtil.java index d570b518c..9a25cb879 100644 --- a/src/main/java/org/broadinstitute/dsm/util/ParticipantUtil.java +++ b/src/main/java/org/broadinstitute/dsm/util/ParticipantUtil.java @@ -4,9 +4,8 @@ import lombok.NonNull; import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; -import org.broadinstitute.dsm.model.elasticsearch.ESProfile; -import org.broadinstitute.dsm.model.elasticsearch.ElasticSearch; -import org.broadinstitute.dsm.model.elasticsearch.ElasticSearchParticipantDto; +import org.broadinstitute.dsm.model.elastic.ESProfile; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import java.util.List; diff --git a/src/test/java/org/broadinstitute/dsm/ElasticSearchTest.java b/src/test/java/org/broadinstitute/dsm/ElasticSearchTest.java index 404697377..a9ef1c6ae 100644 --- a/src/test/java/org/broadinstitute/dsm/ElasticSearchTest.java +++ b/src/test/java/org/broadinstitute/dsm/ElasticSearchTest.java @@ -6,9 +6,8 @@ import org.broadinstitute.ddp.db.TransactionWrapper; import org.broadinstitute.dsm.db.DDPInstance; import org.broadinstitute.dsm.export.WorkflowForES; -import org.broadinstitute.dsm.model.elasticsearch.ESProfile; -import org.broadinstitute.dsm.model.elasticsearch.ElasticSearch; -import org.broadinstitute.dsm.model.elasticsearch.ElasticSearchParticipantDto; +import org.broadinstitute.dsm.model.elastic.ESProfile; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.statics.ApplicationConfigConstants; import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.DBTestUtil; diff --git a/src/test/java/org/broadinstitute/dsm/model/elasticsearch/ElasticSearchTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchTest.java similarity index 95% rename from src/test/java/org/broadinstitute/dsm/model/elasticsearch/ElasticSearchTest.java rename to src/test/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchTest.java index 34ce7d37c..a405faf17 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elasticsearch/ElasticSearchTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchTest.java @@ -1,15 +1,14 @@ -package org.broadinstitute.dsm.model.elasticsearch; +package org.broadinstitute.dsm.model.elastic.search; import java.util.Map; import java.util.Optional; -import com.google.common.reflect.TypeToken; import com.google.gson.Gson; -import org.broadinstitute.dsm.model.participant.ParticipantWrapperDto; +import org.broadinstitute.dsm.model.elastic.ESAddress; +import org.broadinstitute.dsm.model.elastic.ESProfile; import org.broadinstitute.dsm.model.participant.ParticipantWrapperTest; import org.junit.Assert; -import org.junit.BeforeClass; import org.junit.Test; public class ElasticSearchTest { diff --git a/src/test/java/org/broadinstitute/dsm/model/participant/ParticipantWrapperTest.java b/src/test/java/org/broadinstitute/dsm/model/participant/ParticipantWrapperTest.java index 406c6bd8c..890734343 100644 --- a/src/test/java/org/broadinstitute/dsm/model/participant/ParticipantWrapperTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/participant/ParticipantWrapperTest.java @@ -4,18 +4,16 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Random; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.Stream; -import com.google.gson.Gson; import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; -import org.broadinstitute.dsm.model.elasticsearch.ESProfile; -import org.broadinstitute.dsm.model.elasticsearch.ElasticSearch; -import org.broadinstitute.dsm.model.elasticsearch.ElasticSearchParticipantDto; -import org.broadinstitute.dsm.model.elasticsearch.ElasticSearchable; +import org.broadinstitute.dsm.model.elastic.ESProfile; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearchable; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import org.junit.Assert; import org.junit.BeforeClass; diff --git a/src/test/java/org/broadinstitute/dsm/route/KitUploadRouteTest.java b/src/test/java/org/broadinstitute/dsm/route/KitUploadRouteTest.java index ad315436c..890c9af27 100644 --- a/src/test/java/org/broadinstitute/dsm/route/KitUploadRouteTest.java +++ b/src/test/java/org/broadinstitute/dsm/route/KitUploadRouteTest.java @@ -4,9 +4,8 @@ import org.broadinstitute.dsm.TestHelper; import org.broadinstitute.dsm.exception.FileColumnMissing; import org.broadinstitute.dsm.exception.UploadLineException; -import org.broadinstitute.dsm.model.elasticsearch.ESProfile; -import org.broadinstitute.dsm.model.elasticsearch.ElasticSearch; -import org.broadinstitute.dsm.model.elasticsearch.ElasticSearchParticipantDto; +import org.broadinstitute.dsm.model.elastic.ESProfile; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.participant.ParticipantWrapperDto; import org.broadinstitute.dsm.util.NotificationUtil; import org.broadinstitute.dsm.util.ParticipantUtil; From 1edb346c35b3f9c4c858ad818066a14c05dc39bc Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 14 Oct 2021 17:39:59 +0400 Subject: [PATCH 025/405] DDP-7047 | add assembler interface to generate map object for exporter, make some refactoring for patch classes --- .../dsm/model/elastic/export/Assembler.java | 11 +++++++++++ .../dsm/model/patch/AbstractionPatch.java | 4 ++-- .../broadinstitute/dsm/model/patch/BasePatch.java | 14 +++++++------- .../dsm/model/patch/ExistingRecordPatch.java | 11 ++++++++--- .../broadinstitute/dsm/model/patch/NullPatch.java | 4 ++-- .../dsm/model/patch/OncHistoryDetailPatch.java | 4 ++-- .../dsm/model/patch/ParticipantDataPatch.java | 4 ++-- .../dsm/model/patch/ParticipantRecordPatch.java | 4 ++-- .../dsm/model/patch/TissuePatch.java | 4 ++-- 9 files changed, 38 insertions(+), 22 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/Assembler.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Assembler.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/Assembler.java new file mode 100644 index 000000000..869786cb2 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/Assembler.java @@ -0,0 +1,11 @@ +package org.broadinstitute.dsm.model.elastic.export; + +import java.util.Map; + +import org.broadinstitute.dsm.model.NameValue; + +public interface Assembler { + + Map generateSource(NameValue nameValue); + +} diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java index 31deda50d..f64b3c9f3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java @@ -42,13 +42,13 @@ public Object patchNameValuePair() { } @Override - Object handleSingleNameValue(DBElement dbElement) { + Object handleSingleNameValue() { String primaryKeyId = AbstractionWrapper.createNewAbstractionFieldValue(patch.getParentId(), patch.getFieldId(), patch.getUser(), patch.getNameValue(), dbElement); return Map.of(PRIMARY_KEY_ID, primaryKeyId); } @Override - Optional processEachNameValue(NameValue nameValue, DBElement dbElement) { + Optional processEachNameValue(NameValue nameValue) { if (StringUtils.isBlank(primaryKeyId)) { primaryKeyId = AbstractionWrapper.createNewAbstractionFieldValue(patch.getParentId(), patch.getFieldId(), patch.getUser(), nameValue, dbElement); } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index ce9dbf4d7..51011168f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -48,10 +48,10 @@ public abstract class BasePatch { Map resultMap; List nameValues; - protected Patch patch; protected ESProfile profile; protected DDPInstance ddpInstance; + protected DBElement dbElement; { resultMap = new HashMap<>(); @@ -72,7 +72,7 @@ protected BasePatch(Patch patch) { protected abstract Object patchNameValuePair(); - abstract Object handleSingleNameValue(DBElement dbElement); + abstract Object handleSingleNameValue(); private void prepareCommonData() { ddpInstance = DDPInstance.getDDPInstance(patch.getRealm()); @@ -82,9 +82,9 @@ private void prepareCommonData() { Optional processSingleNameValue() { Optional result; - DBElement dbElement = PatchUtil.getColumnNameMap().get(patch.getNameValue().getName()); + dbElement = PatchUtil.getColumnNameMap().get(patch.getNameValue().getName()); if (dbElement != null) { - result = Optional.of(handleSingleNameValue(dbElement)); + result = Optional.of(handleSingleNameValue()); } else { throw new RuntimeException("DBElement not found in ColumnNameMap: " + patch.getNameValue().getName()); @@ -95,9 +95,9 @@ Optional processSingleNameValue() { List processMultipleNameValues() { List updatedNameValues = new ArrayList<>(); for (NameValue nameValue : patch.getNameValues()) { - DBElement dbElement = PatchUtil.getColumnNameMap().get(nameValue.getName()); + dbElement = PatchUtil.getColumnNameMap().get(nameValue.getName()); if (dbElement != null) { - processEachNameValue(nameValue, dbElement).ifPresent(updatedNameValues::add); + processEachNameValue(nameValue).ifPresent(updatedNameValues::add); } else { throw new RuntimeException("DBElement not found in ColumnNameMap: " + nameValue.getName()); @@ -106,7 +106,7 @@ List processMultipleNameValues() { return updatedNameValues; } - abstract Optional processEachNameValue(NameValue nameValue, DBElement dbElement); + abstract Optional processEachNameValue(NameValue nameValue); protected boolean hasQuestion(NameValue nameValue) { return nameValue.getName().contains("question"); diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java index 96d58c0aa..1dd879f79 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java @@ -13,6 +13,7 @@ import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.Value; import org.broadinstitute.dsm.model.elastic.ESProfile; +import org.broadinstitute.dsm.model.elastic.export.Assembler; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import org.broadinstitute.dsm.model.settings.field.FieldSettings; import org.broadinstitute.dsm.statics.DBConstants; @@ -24,7 +25,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class ExistingRecordPatch extends BasePatch { +public class ExistingRecordPatch extends BasePatch implements Assembler { private static final Logger logger = LoggerFactory.getLogger(ExistingRecordPatch.class); @@ -57,7 +58,7 @@ public Object patchNameValuePair() { } @Override - Optional processEachNameValue(NameValue nameValue, DBElement dbElement) { + Optional processEachNameValue(NameValue nameValue) { Optional maybeUpdatedNameValue = Optional.empty(); Patch.patch(patch.getId(), patch.getUser(), nameValue, dbElement); if (hasQuestion(nameValue)) { @@ -174,7 +175,7 @@ else if (EventTypeDao.EVENT.equals(action.getType())) { } @Override - Object handleSingleNameValue(DBElement dbElement) { + Object handleSingleNameValue() { List nameValues = new ArrayList<>(); if (Patch.patch(patch.getId(), patch.getUser(), patch.getNameValue(), dbElement)) { nameValues.addAll(setWorkflowRelatedFields(patch)); @@ -223,4 +224,8 @@ private boolean isMedicalRecord(String name, String type) { return DBConstants.DDP_MEDICAL_RECORD_ALIAS.equals(type) && ESObjectConstants.MEDICAL_RECORDS_FIELD_NAMES.contains(name); } + @Override + public Map generateSource(NameValue nameValue) { + return null; + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java index f422666d1..d27befe65 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/NullPatch.java @@ -23,12 +23,12 @@ public Object patchNameValuePair() { } @Override - Optional processEachNameValue(NameValue nameValue, DBElement dbElement) { + Optional processEachNameValue(NameValue nameValue) { return Optional.empty(); } @Override - Object handleSingleNameValue(DBElement dbElement) { + Object handleSingleNameValue() { return new Object(); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/OncHistoryDetailPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/OncHistoryDetailPatch.java index e542c3516..7a20f162c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/OncHistoryDetailPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/OncHistoryDetailPatch.java @@ -73,7 +73,7 @@ protected Object patchNameValuePair() { } @Override - Object handleSingleNameValue(DBElement dbElement) { + Object handleSingleNameValue() { if (Patch.patch(oncHistoryDetailId, patch.getUser(), patch.getNameValue(), dbElement)) { nameValues.addAll(setWorkflowRelatedFields(patch)); //set oncHistoryDetails created if it is a oncHistoryDetails value without a ID, otherwise created should already be set @@ -89,7 +89,7 @@ Object handleSingleNameValue(DBElement dbElement) { } @Override - Optional processEachNameValue(NameValue nameValue, DBElement dbElement) { + Optional processEachNameValue(NameValue nameValue) { Patch.patch(oncHistoryDetailId, patch.getUser(), nameValue, dbElement); return Optional.empty(); } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantDataPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantDataPatch.java index a41474b0b..325195c54 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantDataPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantDataPatch.java @@ -35,12 +35,12 @@ protected Object patchNameValuePair() { } @Override - Object handleSingleNameValue(DBElement dbElement) { + Object handleSingleNameValue() { return null; } @Override - Optional processEachNameValue(NameValue nameValue, DBElement dbElement) { + Optional processEachNameValue(NameValue nameValue) { if (participantDataId == null) { participantDataId = ParticipantData.createNewParticipantData(patch.getParentId(), ddpInstance.getDdpInstanceId(), patch.getFieldId(), String.valueOf(nameValue.getValue()), patch.getUser()); resultMap.put(ESObjectConstants.PARTICIPANT_DATA_ID, participantDataId); diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantRecordPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantRecordPatch.java index 84192d422..a50a5e50b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantRecordPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantRecordPatch.java @@ -63,14 +63,14 @@ private void insertDdpParticipantRecord(int participantId) { } @Override - Object handleSingleNameValue(DBElement dbElement) { + Object handleSingleNameValue() { Patch.patch(String.valueOf(participantId), patch.getUser(), patch.getNameValue(), dbElement); resultMap.put(PARTICIPANT_ID, String.valueOf(participantId)); return resultMap; } @Override - Optional processEachNameValue(NameValue nameValue, DBElement dbElement) { + Optional processEachNameValue(NameValue nameValue) { return Optional.empty(); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java index a6e2fe931..8fc2e42cb 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java @@ -37,7 +37,7 @@ protected Object patchNameValuePair() { } @Override - Object handleSingleNameValue(DBElement dbElement) { + Object handleSingleNameValue() { if (Patch.patch(tissueId, patch.getUser(), patch.getNameValue(), dbElement)) { nameValues = setWorkflowRelatedFields(patch); resultMap.put(TISSUE_ID, tissueId); @@ -49,7 +49,7 @@ Object handleSingleNameValue(DBElement dbElement) { } @Override - Optional processEachNameValue(NameValue nameValue, DBElement dbElement) { + Optional processEachNameValue(NameValue nameValue) { return Optional.empty(); } } From ce45813d7af1dc69fff5e3a86e0f11d4d36880f9 Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 18 Oct 2021 11:10:00 +0400 Subject: [PATCH 026/405] DDP-7047 | initialize test case for ExistingRecord.patch --- .../dsm/db/structure/DBElement.java | 3 ++ .../dsm/model/patch/ExistingRecordPatch.java | 9 +++- .../model/patch/ExistingRecordPatchTest.java | 42 +++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 src/test/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatchTest.java diff --git a/src/main/java/org/broadinstitute/dsm/db/structure/DBElement.java b/src/main/java/org/broadinstitute/dsm/db/structure/DBElement.java index 058f79941..1d0dc8a9f 100644 --- a/src/main/java/org/broadinstitute/dsm/db/structure/DBElement.java +++ b/src/main/java/org/broadinstitute/dsm/db/structure/DBElement.java @@ -4,12 +4,15 @@ import java.time.Instant; import java.time.ZoneId; import java.time.format.DateTimeFormatter; +import java.util.List; import lombok.Data; @Data public class DBElement { + public static final List JSON_TYPE_COLUMNS = List.of("data", "something", "another thing"); + public String tableName; public String tableAlias; public String primaryKey; diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java index 1dd879f79..cb6f72b83 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java @@ -226,6 +226,13 @@ private boolean isMedicalRecord(String name, String type) { @Override public Map generateSource(NameValue nameValue) { - return null; + Map result = new HashMap<>(); + if (DBElement.JSON_TYPE_COLUMNS.contains(dbElement.getColumnName())) { + Map nestedMap = Map.of(nameValue.getName(), nameValue.getValue()); + result.put(dbElement.getColumnName(), nestedMap); + } else { + result.put(dbElement.getColumnName(), nameValue.getValue()); + } + return result; } } diff --git a/src/test/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatchTest.java b/src/test/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatchTest.java new file mode 100644 index 000000000..75ca9b492 --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatchTest.java @@ -0,0 +1,42 @@ +package org.broadinstitute.dsm.model.patch; + +import org.broadinstitute.dsm.model.NameValue; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class ExistingRecordPatchTest { + + private static Patch patch; + // String id, String parent, String parentId, String user, NameValue nameValue, List nameValues, String ddpParticipantId + + @BeforeClass + public static void setUp() { + String id = "id"; + String parent = "parent"; + String parentId = "parentId"; + String user = "user"; + NameValue nameValue = new NameValue("field", "value"); + int randNumb = new Random().nextInt(11); + List nameValues = Stream.generate(() -> new NameValue("field" + randNumb, "value" + randNumb)) + .limit(10) + .collect(Collectors.toList()); + String ddpParticipantId = "participantId"; + patch = new Patch(id, parent, parentId, user, nameValue, nameValues, ddpParticipantId); + } + + @Test + public void generateSource() { + ExistingRecordPatch existingRecordPatch = new ExistingRecordPatch(patch, null); + Map obj = existingRecordPatch.generateSource(patch.getNameValue()); + Assert.assertEquals("value",obj.get("field")); + } + + +} \ No newline at end of file From ddcfd689b299a9d62397cad6a2050f7e15f69c5d Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 18 Oct 2021 17:11:00 +0400 Subject: [PATCH 027/405] DDP-7047 | change method prepareCommonData access for testing, add payload class for exporting --- .../elastic/export/UpdateRequestPayload.java | 75 +++++++++++++++++++ .../dsm/model/patch/BasePatch.java | 2 +- .../dsm/model/patch/ExistingRecordPatch.java | 2 + .../model/patch/ExistingRecordPatchTest.java | 21 +++++- 4 files changed, 97 insertions(+), 3 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/UpdateRequestPayload.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/UpdateRequestPayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/UpdateRequestPayload.java new file mode 100644 index 000000000..3025e7b5f --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/UpdateRequestPayload.java @@ -0,0 +1,75 @@ +package org.broadinstitute.dsm.model.elastic.export; + +import java.util.Map; + +public class UpdateRequestPayload { + + private String index; + private String type; + private String id; + private Map doc; + private boolean docAsUpsert; + private int retryOnConflict; + + private UpdateRequestPayload(Builder builder) { + this.index = builder.index; + this.type = builder.type; + this.id = builder.id; + this.doc = builder.doc; + this.docAsUpsert = builder.docAsUpsert; + this.retryOnConflict = builder.retryOnConflict; + } + + private static class Builder { + public String index; + public String type; + public String id; + public Map doc; + public boolean docAsUpsert; + public int retryOnConflict; + + public Builder(String index, String type, String id, Map doc) { + this.index = index; + this.type = type; + this.id = id; + this.doc = doc; + } + + public Builder withIndex(String index) { + this.index = index; + return this; + } + + public Builder withType(String type) { + this.type = type; + return this; + } + + public Builder withId(String id) { + this.id = id; + return this; + } + + public Builder withDoc(Map docToUpsert) { + this.doc = docToUpsert; + return this; + } + + public Builder withDocAsUpsert(boolean docAsUpsert) { + this.docAsUpsert = docAsUpsert; + return this; + } + + public Builder withRetryOnConflict(int retryOnConflict) { + this.retryOnConflict = retryOnConflict; + return this; + } + + public UpdateRequestPayload build() { + return new UpdateRequestPayload(this); + } + + + } + +} diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index 51011168f..c82c8ee58 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -74,7 +74,7 @@ protected BasePatch(Patch patch) { abstract Object handleSingleNameValue(); - private void prepareCommonData() { + protected void prepareCommonData() { ddpInstance = DDPInstance.getDDPInstance(patch.getRealm()); profile = ElasticSearchUtil.getParticipantProfileByGuidOrAltPid(ddpInstance.getParticipantIndexES(), patch.getDdpParticipantId()) .orElse(null); diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java index cb6f72b83..0636428c8 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java @@ -20,6 +20,7 @@ import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.broadinstitute.dsm.util.NotificationUtil; +import org.broadinstitute.dsm.util.PatchUtil; import org.json.JSONArray; import org.json.JSONObject; import org.slf4j.Logger; @@ -61,6 +62,7 @@ public Object patchNameValuePair() { Optional processEachNameValue(NameValue nameValue) { Optional maybeUpdatedNameValue = Optional.empty(); Patch.patch(patch.getId(), patch.getUser(), nameValue, dbElement); + Map elasticMapToExport = generateSource(nameValue); if (hasQuestion(nameValue)) { maybeUpdatedNameValue = sendNotificationEmailAndUpdateStatus(patch, nameValue, dbElement); } diff --git a/src/test/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatchTest.java b/src/test/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatchTest.java index 75ca9b492..912f91f36 100644 --- a/src/test/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatchTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatchTest.java @@ -1,6 +1,8 @@ package org.broadinstitute.dsm.model.patch; +import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; +import org.broadinstitute.dsm.util.NotificationUtil; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -33,10 +35,25 @@ public static void setUp() { @Test public void generateSource() { - ExistingRecordPatch existingRecordPatch = new ExistingRecordPatch(patch, null); + ExistingRecordPatch existingRecordPatch = new TestExistingRecordPatch(patch, null); Map obj = existingRecordPatch.generateSource(patch.getNameValue()); - Assert.assertEquals("value",obj.get("field")); + Assert.assertEquals("value", obj.get("field")); + existingRecordPatch.dbElement = new DBElement("tableName", "tbA", "key", "data"); + obj = existingRecordPatch.generateSource(patch.getNameValue()); + Assert.assertEquals("value", ((Map)obj.get("data")).get("field")); } + private static class TestExistingRecordPatch extends ExistingRecordPatch { + + public TestExistingRecordPatch(Patch patch, NotificationUtil notificationUtil) { + super(patch, notificationUtil); + dbElement = new DBElement("tableName", "tbA", "key", "field"); + } + + @Override + protected void prepareCommonData() { + + } + } } \ No newline at end of file From d53ff2b18268a0f87a5b22cd73fe2e45d5f20a30 Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 18 Oct 2021 18:30:37 +0400 Subject: [PATCH 028/405] DDP-7047 | introduce BaseExporter and ElasticExporter classes which are responsible for implementing export() method that sends data from server into ES --- .../model/elastic/export/BaseExporter.java | 10 +++++ .../elastic/export/ElasticExportAdapter.java | 25 +++++++++++++ .../elastic/export/UpdateRequestPayload.java | 37 +++++++++++-------- .../dsm/model/patch/BasePatch.java | 7 ++++ .../dsm/model/patch/ExistingRecordPatch.java | 2 + .../dsm/model/patch/PatchFactory.java | 2 + 6 files changed, 67 insertions(+), 16 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticExportAdapter.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java new file mode 100644 index 000000000..be864949a --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java @@ -0,0 +1,10 @@ +package org.broadinstitute.dsm.model.elastic.export; + +public abstract class BaseExporter implements Exportable { + + protected UpdateRequestPayload updateRequestPayload; + + protected void setUpdateRequestPayload(UpdateRequestPayload updateRequestPayload) { + this.updateRequestPayload = updateRequestPayload; + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticExportAdapter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticExportAdapter.java new file mode 100644 index 000000000..be9d0e71a --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticExportAdapter.java @@ -0,0 +1,25 @@ +package org.broadinstitute.dsm.model.elastic.export; + + +import org.broadinstitute.dsm.util.ElasticSearchUtil; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; + +import java.io.IOException; +import java.util.Map; +import java.util.Objects; + +public class ElasticExportAdapter extends BaseExporter { + + @Override + public void export(Map data) { + RestHighLevelClient clientInstance = ElasticSearchUtil.getClientInstance(); + UpdateRequest updateRequest = Objects.requireNonNull(updateRequestPayload).getUpdateRequest(data); + try { + clientInstance.update(updateRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + throw new RuntimeException("Error while exporting data to ES", e); + } + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/UpdateRequestPayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/UpdateRequestPayload.java index 3025e7b5f..635d484c1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/UpdateRequestPayload.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/UpdateRequestPayload.java @@ -1,13 +1,16 @@ package org.broadinstitute.dsm.model.elastic.export; +import org.elasticsearch.action.update.UpdateRequest; + import java.util.Map; public class UpdateRequestPayload { + private static final String DOC = "_doc"; + private String index; private String type; private String id; - private Map doc; private boolean docAsUpsert; private int retryOnConflict; @@ -15,24 +18,31 @@ private UpdateRequestPayload(Builder builder) { this.index = builder.index; this.type = builder.type; this.id = builder.id; - this.doc = builder.doc; this.docAsUpsert = builder.docAsUpsert; this.retryOnConflict = builder.retryOnConflict; } + public UpdateRequest getUpdateRequest(Map data) { + return new UpdateRequest() + .index(index) + .type(DOC) + .id(id) + .doc(data) + .docAsUpsert(docAsUpsert) + .retryOnConflict(retryOnConflict); + } + private static class Builder { - public String index; - public String type; - public String id; - public Map doc; - public boolean docAsUpsert; - public int retryOnConflict; - - public Builder(String index, String type, String id, Map doc) { + private String index; + private String type; + private String id; + private boolean docAsUpsert = true; + private int retryOnConflict; + + public Builder(String index, String type, String id) { this.index = index; this.type = type; this.id = id; - this.doc = doc; } public Builder withIndex(String index) { @@ -50,11 +60,6 @@ public Builder withId(String id) { return this; } - public Builder withDoc(Map docToUpsert) { - this.doc = docToUpsert; - return this; - } - public Builder withDocAsUpsert(boolean docAsUpsert) { this.docAsUpsert = docAsUpsert; return this; diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index c82c8ee58..2848162ba 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -24,6 +24,7 @@ import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.Value; import org.broadinstitute.dsm.model.elastic.ESProfile; +import org.broadinstitute.dsm.model.elastic.export.Exportable; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; @@ -52,6 +53,8 @@ public abstract class BasePatch { protected ESProfile profile; protected DDPInstance ddpInstance; protected DBElement dbElement; + protected Exportable exportable; + { resultMap = new HashMap<>(); @@ -257,4 +260,8 @@ private NameValue setAdditionalValue(String additionalValue, @NonNull Patch patc } } + public void setExportable(Exportable exportable) { + this.exportable = exportable; + } + } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java index 0636428c8..b3e377eb6 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java @@ -63,6 +63,8 @@ Optional processEachNameValue(NameValue nameValue) { Optional maybeUpdatedNameValue = Optional.empty(); Patch.patch(patch.getId(), patch.getUser(), nameValue, dbElement); Map elasticMapToExport = generateSource(nameValue); + RameClass export = ImplementorRameClass(); + export.export(updateRequestPayload); if (hasQuestion(nameValue)) { maybeUpdatedNameValue = sendNotificationEmailAndUpdateStatus(patch, nameValue, dbElement); } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java index bc34f14a8..7132dac0d 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java @@ -1,6 +1,7 @@ package org.broadinstitute.dsm.model.patch; import org.apache.commons.lang3.StringUtils; +import org.broadinstitute.dsm.model.elastic.export.ElasticExportAdapter; import org.broadinstitute.dsm.util.NotificationUtil; public class PatchFactory { @@ -24,6 +25,7 @@ public static BasePatch makePatch(Patch patch, NotificationUtil notificationUtil patcher = new ParticipantRecordPatch(patch); } } + patcher.setExportable(new ElasticExportAdapter()); if (patcher instanceof NullPatch) { throw new RuntimeException("Id and parentId was null"); } From a765a3a9aae8191a4c06d0e3fb14af8a788d5127 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 18 Oct 2021 19:12:02 +0400 Subject: [PATCH 029/405] DDP-7047 | minor refactoring, extract method for export es for generalization --- .../model/elastic/export/BaseExporter.java | 2 +- .../elastic/export/ElasticExportAdapter.java | 2 +- .../elastic/export/UpdateRequestPayload.java | 5 ++-- .../dsm/model/patch/BasePatch.java | 23 ++++++++++++++++--- .../dsm/model/patch/ExistingRecordPatch.java | 8 +++---- 5 files changed, 27 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java index be864949a..dabc1c440 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java @@ -4,7 +4,7 @@ public abstract class BaseExporter implements Exportable { protected UpdateRequestPayload updateRequestPayload; - protected void setUpdateRequestPayload(UpdateRequestPayload updateRequestPayload) { + public void setUpdateRequestPayload(UpdateRequestPayload updateRequestPayload) { this.updateRequestPayload = updateRequestPayload; } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticExportAdapter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticExportAdapter.java index be9d0e71a..ab81e1836 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticExportAdapter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticExportAdapter.java @@ -10,7 +10,7 @@ import java.util.Map; import java.util.Objects; -public class ElasticExportAdapter extends BaseExporter { +public class ElasticExportAdapter extends BaseExporter { @Override public void export(Map data) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/UpdateRequestPayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/UpdateRequestPayload.java index 635d484c1..7463e0caf 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/UpdateRequestPayload.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/UpdateRequestPayload.java @@ -32,16 +32,15 @@ public UpdateRequest getUpdateRequest(Map data) { .retryOnConflict(retryOnConflict); } - private static class Builder { + public static class Builder { private String index; private String type; private String id; private boolean docAsUpsert = true; private int retryOnConflict; - public Builder(String index, String type, String id) { + public Builder(String index, String id) { this.index = index; - this.type = type; this.id = id; } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index 2848162ba..9e56cf040 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -24,7 +24,10 @@ import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.Value; import org.broadinstitute.dsm.model.elastic.ESProfile; +import org.broadinstitute.dsm.model.elastic.export.Assembler; +import org.broadinstitute.dsm.model.elastic.export.BaseExporter; import org.broadinstitute.dsm.model.elastic.export.Exportable; +import org.broadinstitute.dsm.model.elastic.export.UpdateRequestPayload; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; @@ -35,7 +38,7 @@ import org.slf4j.LoggerFactory; -public abstract class BasePatch { +public abstract class BasePatch implements Assembler { static final Logger logger = LoggerFactory.getLogger(BasePatch.class); @@ -53,7 +56,7 @@ public abstract class BasePatch { protected ESProfile profile; protected DDPInstance ddpInstance; protected DBElement dbElement; - protected Exportable exportable; + protected BaseExporter exportable; { @@ -69,6 +72,16 @@ protected BasePatch(Patch patch) { prepareCommonData(); } + protected void exportToES(NameValue nameValue) { + Map elasticMapToExport = generateSource(nameValue); + UpdateRequestPayload updateRequestPayload = new UpdateRequestPayload.Builder(ddpInstance.getParticipantIndexES(), patch.getDdpParticipantId()) + .withDocAsUpsert(true) + .withRetryOnConflict(5) + .build(); + exportable.setUpdateRequestPayload(updateRequestPayload); + exportable.export(elasticMapToExport); + } + public abstract Object doPatch(); protected abstract Object patchNameValuePairs(); @@ -260,8 +273,12 @@ private NameValue setAdditionalValue(String additionalValue, @NonNull Patch patc } } - public void setExportable(Exportable exportable) { + public void setExportable(BaseExporter exportable) { this.exportable = exportable; } + @Override + public Map generateSource(NameValue nameValue) { + return Map.of(); + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java index b3e377eb6..c92def096 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java @@ -20,13 +20,12 @@ import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.broadinstitute.dsm.util.NotificationUtil; -import org.broadinstitute.dsm.util.PatchUtil; import org.json.JSONArray; import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class ExistingRecordPatch extends BasePatch implements Assembler { +public class ExistingRecordPatch extends BasePatch { private static final Logger logger = LoggerFactory.getLogger(ExistingRecordPatch.class); @@ -62,9 +61,7 @@ public Object patchNameValuePair() { Optional processEachNameValue(NameValue nameValue) { Optional maybeUpdatedNameValue = Optional.empty(); Patch.patch(patch.getId(), patch.getUser(), nameValue, dbElement); - Map elasticMapToExport = generateSource(nameValue); - RameClass export = ImplementorRameClass(); - export.export(updateRequestPayload); + exportToES(nameValue); if (hasQuestion(nameValue)) { maybeUpdatedNameValue = sendNotificationEmailAndUpdateStatus(patch, nameValue, dbElement); } @@ -184,6 +181,7 @@ Object handleSingleNameValue() { if (Patch.patch(patch.getId(), patch.getUser(), patch.getNameValue(), dbElement)) { nameValues.addAll(setWorkflowRelatedFields(patch)); writeDSMRecordsToES(); + exportToES(patch.getNameValue()); //return nameValues with nulls return nameValues; } From 599284a3b390d91ca9b354f8990923a2c56af7fa Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 19 Oct 2021 16:46:31 +0400 Subject: [PATCH 030/405] DDP-7047 | introduce Generator interface which will be responsible for generating appropriate input data which is necessary in order to make UpdateRequest in ES --- .../export/{Assembler.java => Generator.java} | 6 +-- .../model/elastic/export/SourceGenerator.java | 47 +++++++++++++++++++ .../dsm/model/patch/BasePatch.java | 9 ++-- .../dsm/model/patch/ExistingRecordPatch.java | 7 ++- .../elastic/export/SourceGeneratorTest.java | 9 ++++ .../model/patch/ExistingRecordPatchTest.java | 4 +- 6 files changed, 67 insertions(+), 15 deletions(-) rename src/main/java/org/broadinstitute/dsm/model/elastic/export/{Assembler.java => Generator.java} (57%) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Assembler.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/Generator.java similarity index 57% rename from src/main/java/org/broadinstitute/dsm/model/elastic/export/Assembler.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/export/Generator.java index 869786cb2..75c0907f3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Assembler.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/Generator.java @@ -4,8 +4,6 @@ import org.broadinstitute.dsm.model.NameValue; -public interface Assembler { - - Map generateSource(NameValue nameValue); - +public interface Generator { + Map generate(NameValue nameValue); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java new file mode 100644 index 000000000..7847cca7b --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java @@ -0,0 +1,47 @@ +package org.broadinstitute.dsm.model.elastic.export; + +import org.broadinstitute.dsm.db.structure.DBElement; +import org.broadinstitute.dsm.model.NameValue; +import org.broadinstitute.dsm.util.PatchUtil; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +public class SourceGenerator implements Generator { + + public static final String DSM_OBJECT = "dsm"; + public static final Map TABLE_ALIAS_MAPPINGS = Map.of( + "m", "medicalRecords", + "t", "tissueRecords", + "oD", "oncHistoryDetailRecords" + ); + + private PatchUtil patchUtil; + + public SourceGenerator(PatchUtil patchUtil) { + this.patchUtil = patchUtil; + } + + /* + "r", "unknown", + "p", "unknown" + * */ + + @Override + public Map generate(NameValue nameValue) { + DBElement dbElement = patchUtil.getColumnNameMap().get(Objects.requireNonNull(nameValue).getName()); + + Map result = new HashMap<>(); + String objectKey = TABLE_ALIAS_MAPPINGS.get(dbElement.getTableAlias()); + + if (DBElement.JSON_TYPE_COLUMNS.contains(dbElement.getColumnName())) { + Map nestedMap = Map.of(nameValue.getName(), nameValue.getValue()); + result.put(objectKey, Map.of(dbElement.getColumnName(), nestedMap)); + } else { + result.put(objectKey, Map.of(dbElement.getColumnName(), nameValue.getValue())); + } + + return Map.of(DSM_OBJECT, result); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index 9e56cf040..6b7b0595a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -24,9 +24,8 @@ import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.Value; import org.broadinstitute.dsm.model.elastic.ESProfile; -import org.broadinstitute.dsm.model.elastic.export.Assembler; +import org.broadinstitute.dsm.model.elastic.export.Generator; import org.broadinstitute.dsm.model.elastic.export.BaseExporter; -import org.broadinstitute.dsm.model.elastic.export.Exportable; import org.broadinstitute.dsm.model.elastic.export.UpdateRequestPayload; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import org.broadinstitute.dsm.statics.ESObjectConstants; @@ -38,7 +37,7 @@ import org.slf4j.LoggerFactory; -public abstract class BasePatch implements Assembler { +public abstract class BasePatch implements Generator { static final Logger logger = LoggerFactory.getLogger(BasePatch.class); @@ -73,7 +72,7 @@ protected BasePatch(Patch patch) { } protected void exportToES(NameValue nameValue) { - Map elasticMapToExport = generateSource(nameValue); + Map elasticMapToExport = generate(nameValue); UpdateRequestPayload updateRequestPayload = new UpdateRequestPayload.Builder(ddpInstance.getParticipantIndexES(), patch.getDdpParticipantId()) .withDocAsUpsert(true) .withRetryOnConflict(5) @@ -278,7 +277,7 @@ public void setExportable(BaseExporter exportable) { } @Override - public Map generateSource(NameValue nameValue) { + public Map generate(NameValue nameValue) { return Map.of(); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java index c92def096..da1709dda 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java @@ -13,7 +13,6 @@ import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.Value; import org.broadinstitute.dsm.model.elastic.ESProfile; -import org.broadinstitute.dsm.model.elastic.export.Assembler; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import org.broadinstitute.dsm.model.settings.field.FieldSettings; import org.broadinstitute.dsm.statics.DBConstants; @@ -58,7 +57,7 @@ public Object patchNameValuePair() { } @Override - Optional processEachNameValue(NameValue nameValue) { +Optional processEachNameValue(NameValue nameValue) { Optional maybeUpdatedNameValue = Optional.empty(); Patch.patch(patch.getId(), patch.getUser(), nameValue, dbElement); exportToES(nameValue); @@ -180,7 +179,7 @@ Object handleSingleNameValue() { List nameValues = new ArrayList<>(); if (Patch.patch(patch.getId(), patch.getUser(), patch.getNameValue(), dbElement)) { nameValues.addAll(setWorkflowRelatedFields(patch)); - writeDSMRecordsToES(); +// writeDSMRecordsToES(); exportToES(patch.getNameValue()); //return nameValues with nulls return nameValues; @@ -227,7 +226,7 @@ private boolean isMedicalRecord(String name, String type) { } @Override - public Map generateSource(NameValue nameValue) { + public Map generate(NameValue nameValue) { Map result = new HashMap<>(); if (DBElement.JSON_TYPE_COLUMNS.contains(dbElement.getColumnName())) { Map nestedMap = Map.of(nameValue.getName(), nameValue.getValue()); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java new file mode 100644 index 000000000..a2677ec00 --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java @@ -0,0 +1,9 @@ +package org.broadinstitute.dsm.model.elastic.export; + +import junit.framework.TestCase; + +public class SourceGeneratorTest extends TestCase { + + + +} \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatchTest.java b/src/test/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatchTest.java index 912f91f36..c5d4f5365 100644 --- a/src/test/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatchTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatchTest.java @@ -36,10 +36,10 @@ public static void setUp() { @Test public void generateSource() { ExistingRecordPatch existingRecordPatch = new TestExistingRecordPatch(patch, null); - Map obj = existingRecordPatch.generateSource(patch.getNameValue()); + Map obj = existingRecordPatch.generate(patch.getNameValue()); Assert.assertEquals("value", obj.get("field")); existingRecordPatch.dbElement = new DBElement("tableName", "tbA", "key", "data"); - obj = existingRecordPatch.generateSource(patch.getNameValue()); + obj = existingRecordPatch.generate(patch.getNameValue()); Assert.assertEquals("value", ((Map)obj.get("data")).get("field")); } From a0e7dbeaee848761320f197b5592142c17faa727 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 19 Oct 2021 17:41:06 +0400 Subject: [PATCH 031/405] DDP-7047 | test for generate method --- .../elastic/export/SourceGeneratorTest.java | 24 +++++++++++++++---- .../model/elastic/export/TestPatchUtil.java | 15 ++++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/export/TestPatchUtil.java diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java index a2677ec00..48ed9e969 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java @@ -1,9 +1,25 @@ package org.broadinstitute.dsm.model.elastic.export; -import junit.framework.TestCase; +import java.util.Map; -public class SourceGeneratorTest extends TestCase { +import org.broadinstitute.dsm.db.structure.DBElement; +import org.broadinstitute.dsm.model.NameValue; +import org.broadinstitute.dsm.util.PatchUtil; +import org.junit.Test; +import org.junit.Assert; - +public class SourceGeneratorTest { + + @Test + public void generate() { + Generator generator = new SourceGenerator(new TestPatchUtil()); + NameValue nameValue = new NameValue("medical_record_column", "value"); + Map objectMap = generator.generate(nameValue); + Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), SourceGenerator.DSM_OBJECT); + String value = (String)((Map)((Map)objectMap.get(SourceGenerator.DSM_OBJECT)).get("m")).get("medical_record_column"); + Assert.assertEquals("value", value); + } + + +} -} \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/TestPatchUtil.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/TestPatchUtil.java new file mode 100644 index 000000000..45d6d75f5 --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/TestPatchUtil.java @@ -0,0 +1,15 @@ +package org.broadinstitute.dsm.model.elastic.export; + +import java.util.Map; + +import org.broadinstitute.dsm.db.structure.DBElement; +import org.broadinstitute.dsm.util.PatchUtil; + +public class TestPatchUtil extends PatchUtil { + + public static Map getColumnNameMap() { + DBElement dbElement = new DBElement("ddp_medical_record", "m", "pr", "medical_record_column"); + return Map.of("medical_record_name", dbElement); + } + +} From 4576e17e8adbfd000b888b6bead74e0f8bcd205a Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 19 Oct 2021 17:57:21 +0400 Subject: [PATCH 032/405] DDP-7047 | refactor test case for SourceGenerator --- .../dsm/model/elastic/export/SourceGenerator.java | 12 +++++------- .../model/elastic/export/SourceGeneratorTest.java | 8 +++++++- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java index 7847cca7b..d8abc204d 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java @@ -17,12 +17,6 @@ public class SourceGenerator implements Generator { "oD", "oncHistoryDetailRecords" ); - private PatchUtil patchUtil; - - public SourceGenerator(PatchUtil patchUtil) { - this.patchUtil = patchUtil; - } - /* "r", "unknown", "p", "unknown" @@ -30,7 +24,7 @@ public SourceGenerator(PatchUtil patchUtil) { @Override public Map generate(NameValue nameValue) { - DBElement dbElement = patchUtil.getColumnNameMap().get(Objects.requireNonNull(nameValue).getName()); + DBElement dbElement = getDBElement(nameValue); Map result = new HashMap<>(); String objectKey = TABLE_ALIAS_MAPPINGS.get(dbElement.getTableAlias()); @@ -44,4 +38,8 @@ public Map generate(NameValue nameValue) { return Map.of(DSM_OBJECT, result); } + + protected DBElement getDBElement(NameValue nameValue) { + return PatchUtil.getColumnNameMap().get(Objects.requireNonNull(nameValue).getName()); + } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java index 48ed9e969..ee600fcf6 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java @@ -12,7 +12,7 @@ public class SourceGeneratorTest { @Test public void generate() { - Generator generator = new SourceGenerator(new TestPatchUtil()); + Generator generator = new TestSourceGenerator(); NameValue nameValue = new NameValue("medical_record_column", "value"); Map objectMap = generator.generate(nameValue); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), SourceGenerator.DSM_OBJECT); @@ -20,6 +20,12 @@ public void generate() { Assert.assertEquals("value", value); } + private static class TestSourceGenerator extends SourceGenerator { + @Override + protected DBElement getDBElement(NameValue nameValue) { + return TestPatchUtil.getColumnNameMap().get(nameValue.getName()); + } + } } From 50afbeb30792360e2ad52b9fd9382a863182c923 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 19 Oct 2021 18:06:36 +0400 Subject: [PATCH 033/405] DDP-7047 | fix the test --- .../model/elastic/export/SourceGeneratorTest.java | 11 ++++++++++- .../dsm/model/elastic/export/TestPatchUtil.java | 15 --------------- 2 files changed, 10 insertions(+), 16 deletions(-) delete mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/export/TestPatchUtil.java diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java index ee600fcf6..9aaccb338 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java @@ -16,7 +16,7 @@ public void generate() { NameValue nameValue = new NameValue("medical_record_column", "value"); Map objectMap = generator.generate(nameValue); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), SourceGenerator.DSM_OBJECT); - String value = (String)((Map)((Map)objectMap.get(SourceGenerator.DSM_OBJECT)).get("m")).get("medical_record_column"); + String value = (String)((Map)((Map)objectMap.get(SourceGenerator.DSM_OBJECT)).get("medicalRecords")).get("medical_record_column"); Assert.assertEquals("value", value); } @@ -27,5 +27,14 @@ protected DBElement getDBElement(NameValue nameValue) { return TestPatchUtil.getColumnNameMap().get(nameValue.getName()); } } + + private static class TestPatchUtil extends PatchUtil { + + public static Map getColumnNameMap() { + DBElement dbElement = new DBElement("ddp_medical_record", "m", "pr", "medical_record_column"); + return Map.of("medical_record_column", dbElement); + } + + } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/TestPatchUtil.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/TestPatchUtil.java deleted file mode 100644 index 45d6d75f5..000000000 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/TestPatchUtil.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.broadinstitute.dsm.model.elastic.export; - -import java.util.Map; - -import org.broadinstitute.dsm.db.structure.DBElement; -import org.broadinstitute.dsm.util.PatchUtil; - -public class TestPatchUtil extends PatchUtil { - - public static Map getColumnNameMap() { - DBElement dbElement = new DBElement("ddp_medical_record", "m", "pr", "medical_record_column"); - return Map.of("medical_record_name", dbElement); - } - -} From a99b1f0e02b029a6d316ad66f89cc532d3faf616 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 19 Oct 2021 18:37:00 +0400 Subject: [PATCH 034/405] DDP-7047 | extract constant for generator test --- .../dsm/model/elastic/export/SourceGeneratorTest.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java index 9aaccb338..b4e38dc43 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java @@ -10,13 +10,15 @@ public class SourceGeneratorTest { + public static final String MEDICAL_RECORD_COLUMN = "medical_record_column"; + @Test public void generate() { Generator generator = new TestSourceGenerator(); - NameValue nameValue = new NameValue("medical_record_column", "value"); + NameValue nameValue = new NameValue(MEDICAL_RECORD_COLUMN, "value"); Map objectMap = generator.generate(nameValue); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), SourceGenerator.DSM_OBJECT); - String value = (String)((Map)((Map)objectMap.get(SourceGenerator.DSM_OBJECT)).get("medicalRecords")).get("medical_record_column"); + String value = (String)((Map)((Map)objectMap.get(SourceGenerator.DSM_OBJECT)).get("medicalRecords")).get(MEDICAL_RECORD_COLUMN); Assert.assertEquals("value", value); } @@ -31,8 +33,8 @@ protected DBElement getDBElement(NameValue nameValue) { private static class TestPatchUtil extends PatchUtil { public static Map getColumnNameMap() { - DBElement dbElement = new DBElement("ddp_medical_record", "m", "pr", "medical_record_column"); - return Map.of("medical_record_column", dbElement); + DBElement dbElement = new DBElement("ddp_medical_record", "m", "pr", MEDICAL_RECORD_COLUMN); + return Map.of(MEDICAL_RECORD_COLUMN, dbElement); } } From b2d297cf1619b4b71874a99ebae9d1eb27b6a5de Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 19 Oct 2021 19:11:53 +0400 Subject: [PATCH 035/405] DDP-7047 | further refactoring --- .../dsm/model/patch/BasePatch.java | 14 ++--- .../dsm/model/patch/ExistingRecordPatch.java | 12 ---- .../dsm/model/patch/PatchFactory.java | 2 + .../model/patch/ExistingRecordPatchTest.java | 59 ------------------- 4 files changed, 9 insertions(+), 78 deletions(-) delete mode 100644 src/test/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatchTest.java diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index 6b7b0595a..71d5bef6a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -37,7 +37,7 @@ import org.slf4j.LoggerFactory; -public abstract class BasePatch implements Generator { +public abstract class BasePatch { static final Logger logger = LoggerFactory.getLogger(BasePatch.class); @@ -56,7 +56,8 @@ public abstract class BasePatch implements Generator { protected DDPInstance ddpInstance; protected DBElement dbElement; protected BaseExporter exportable; - + protected Generator generator; + { resultMap = new HashMap<>(); @@ -72,7 +73,7 @@ protected BasePatch(Patch patch) { } protected void exportToES(NameValue nameValue) { - Map elasticMapToExport = generate(nameValue); + Map elasticMapToExport = generator.generate(nameValue); UpdateRequestPayload updateRequestPayload = new UpdateRequestPayload.Builder(ddpInstance.getParticipantIndexES(), patch.getDdpParticipantId()) .withDocAsUpsert(true) .withRetryOnConflict(5) @@ -276,8 +277,7 @@ public void setExportable(BaseExporter exportable) { this.exportable = exportable; } - @Override - public Map generate(NameValue nameValue) { - return Map.of(); - } + public void setGenerator(Generator generator) { + this.generator = generator; } + } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java index da1709dda..b0cd88fad 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java @@ -224,16 +224,4 @@ private String getTypeFrom(NameValue nameValue) { private boolean isMedicalRecord(String name, String type) { return DBConstants.DDP_MEDICAL_RECORD_ALIAS.equals(type) && ESObjectConstants.MEDICAL_RECORDS_FIELD_NAMES.contains(name); } - - @Override - public Map generate(NameValue nameValue) { - Map result = new HashMap<>(); - if (DBElement.JSON_TYPE_COLUMNS.contains(dbElement.getColumnName())) { - Map nestedMap = Map.of(nameValue.getName(), nameValue.getValue()); - result.put(dbElement.getColumnName(), nestedMap); - } else { - result.put(dbElement.getColumnName(), nameValue.getValue()); - } - return result; - } } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java index 7132dac0d..1586d008a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java @@ -2,6 +2,7 @@ import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.elastic.export.ElasticExportAdapter; +import org.broadinstitute.dsm.model.elastic.export.SourceGenerator; import org.broadinstitute.dsm.util.NotificationUtil; public class PatchFactory { @@ -26,6 +27,7 @@ public static BasePatch makePatch(Patch patch, NotificationUtil notificationUtil } } patcher.setExportable(new ElasticExportAdapter()); + patcher.setGenerator(new SourceGenerator()); if (patcher instanceof NullPatch) { throw new RuntimeException("Id and parentId was null"); } diff --git a/src/test/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatchTest.java b/src/test/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatchTest.java deleted file mode 100644 index c5d4f5365..000000000 --- a/src/test/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatchTest.java +++ /dev/null @@ -1,59 +0,0 @@ -package org.broadinstitute.dsm.model.patch; - -import org.broadinstitute.dsm.db.structure.DBElement; -import org.broadinstitute.dsm.model.NameValue; -import org.broadinstitute.dsm.util.NotificationUtil; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -public class ExistingRecordPatchTest { - - private static Patch patch; - // String id, String parent, String parentId, String user, NameValue nameValue, List nameValues, String ddpParticipantId - - @BeforeClass - public static void setUp() { - String id = "id"; - String parent = "parent"; - String parentId = "parentId"; - String user = "user"; - NameValue nameValue = new NameValue("field", "value"); - int randNumb = new Random().nextInt(11); - List nameValues = Stream.generate(() -> new NameValue("field" + randNumb, "value" + randNumb)) - .limit(10) - .collect(Collectors.toList()); - String ddpParticipantId = "participantId"; - patch = new Patch(id, parent, parentId, user, nameValue, nameValues, ddpParticipantId); - } - - @Test - public void generateSource() { - ExistingRecordPatch existingRecordPatch = new TestExistingRecordPatch(patch, null); - Map obj = existingRecordPatch.generate(patch.getNameValue()); - Assert.assertEquals("value", obj.get("field")); - existingRecordPatch.dbElement = new DBElement("tableName", "tbA", "key", "data"); - obj = existingRecordPatch.generate(patch.getNameValue()); - Assert.assertEquals("value", ((Map)obj.get("data")).get("field")); - } - - private static class TestExistingRecordPatch extends ExistingRecordPatch { - - public TestExistingRecordPatch(Patch patch, NotificationUtil notificationUtil) { - super(patch, notificationUtil); - dbElement = new DBElement("tableName", "tbA", "key", "field"); - } - - @Override - protected void prepareCommonData() { - - } - } - -} \ No newline at end of file From b5bd5bf11b85aec23351244f0e4ef5337dd2a2a2 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 19 Oct 2021 19:42:39 +0400 Subject: [PATCH 036/405] DDP-7047 | add mappings --- .../model/elastic/export/SourceGenerator.java | 26 +++++++++---------- .../dsm/model/patch/ExistingRecordPatch.java | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java index d8abc204d..3b99077ae 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java @@ -1,5 +1,7 @@ package org.broadinstitute.dsm.model.elastic.export; +import com.google.common.reflect.TypeToken; +import com.google.gson.Gson; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.util.PatchUtil; @@ -14,27 +16,25 @@ public class SourceGenerator implements Generator { public static final Map TABLE_ALIAS_MAPPINGS = Map.of( "m", "medicalRecords", "t", "tissueRecords", - "oD", "oncHistoryDetailRecords" + "oD", "oncHistoryDetailRecords", + "r", "participant", + "p", "participant", + "d", "participant" ); - /* - "r", "unknown", - "p", "unknown" - * */ - @Override public Map generate(NameValue nameValue) { DBElement dbElement = getDBElement(nameValue); Map result = new HashMap<>(); String objectKey = TABLE_ALIAS_MAPPINGS.get(dbElement.getTableAlias()); - - if (DBElement.JSON_TYPE_COLUMNS.contains(dbElement.getColumnName())) { - Map nestedMap = Map.of(nameValue.getName(), nameValue.getValue()); - result.put(objectKey, Map.of(dbElement.getColumnName(), nestedMap)); - } else { - result.put(objectKey, Map.of(dbElement.getColumnName(), nameValue.getValue())); - } + result.put(objectKey, Map.of(dbElement.getColumnName(), nameValue.getValue())); + Map map = new Gson().fromJson((String) nameValue.getValue(), Map.class); +// if (DBElement.JSON_TYPE_COLUMNS.contains(dbElement.getColumnName())) { +// Map nestedMap = Map.of(nameValue.getName(), nameValue.getValue()); +// result.put(objectKey, Map.of(dbElement.getColumnName(), nestedMap)); +// } else { +// } return Map.of(DSM_OBJECT, result); } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java index b0cd88fad..b4618c82c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java @@ -57,7 +57,7 @@ public Object patchNameValuePair() { } @Override -Optional processEachNameValue(NameValue nameValue) { + Optional processEachNameValue(NameValue nameValue) { Optional maybeUpdatedNameValue = Optional.empty(); Patch.patch(patch.getId(), patch.getUser(), nameValue, dbElement); exportToES(nameValue); From d30c7f0221717016bdd3653acada18c09b153717 Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 20 Oct 2021 10:38:12 +0400 Subject: [PATCH 037/405] DDP-7047 | refactor code associated with SourceGenerator alongside with its' test --- .../model/elastic/export/SourceGenerator.java | 34 ++++++++++++------- .../elastic/export/SourceGeneratorTest.java | 2 +- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java index 3b99077ae..a5e9d9b00 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java @@ -1,7 +1,7 @@ package org.broadinstitute.dsm.model.elastic.export; -import com.google.common.reflect.TypeToken; import com.google.gson.Gson; +import com.google.gson.JsonSyntaxException; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.util.PatchUtil; @@ -21,25 +21,35 @@ public class SourceGenerator implements Generator { "p", "participant", "d", "participant" ); + protected NameValue nameValue; + protected DBElement dbElement; @Override public Map generate(NameValue nameValue) { - DBElement dbElement = getDBElement(nameValue); + initializeNecessaryFields(nameValue); + Map mapToExport = collectExportData(); + return Map.of(DSM_OBJECT, mapToExport); + } + + protected void initializeNecessaryFields(NameValue nameValue) { + this.nameValue = nameValue; + dbElement = getDBElement(); + } + protected Map collectExportData() { Map result = new HashMap<>(); String objectKey = TABLE_ALIAS_MAPPINGS.get(dbElement.getTableAlias()); - result.put(objectKey, Map.of(dbElement.getColumnName(), nameValue.getValue())); - Map map = new Gson().fromJson((String) nameValue.getValue(), Map.class); -// if (DBElement.JSON_TYPE_COLUMNS.contains(dbElement.getColumnName())) { -// Map nestedMap = Map.of(nameValue.getName(), nameValue.getValue()); -// result.put(objectKey, Map.of(dbElement.getColumnName(), nestedMap)); -// } else { -// } - - return Map.of(DSM_OBJECT, result); + Map dynamicFieldValues; + try { + dynamicFieldValues = new Gson().fromJson((String) nameValue.getValue(), Map.class); + result.put(objectKey, dynamicFieldValues); + } catch (JsonSyntaxException jse) { + result.put(objectKey, Map.of(dbElement.getColumnName(), nameValue.getValue())); + } + return result; } - protected DBElement getDBElement(NameValue nameValue) { + protected DBElement getDBElement() { return PatchUtil.getColumnNameMap().get(Objects.requireNonNull(nameValue).getName()); } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java index b4e38dc43..a65a1ff43 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java @@ -25,7 +25,7 @@ public void generate() { private static class TestSourceGenerator extends SourceGenerator { @Override - protected DBElement getDBElement(NameValue nameValue) { + protected DBElement getDBElement() { return TestPatchUtil.getColumnNameMap().get(nameValue.getName()); } } From 12c61e3d309ff5ee8d4e4b5ddeee7298e57c8296 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 20 Oct 2021 11:09:57 +0400 Subject: [PATCH 038/405] DDP-7047 | add null check with help of encpsulation to avoid possible NPEs in future --- .../dsm/model/elastic/export/SourceGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java index a5e9d9b00..5e047973b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java @@ -26,7 +26,7 @@ public class SourceGenerator implements Generator { @Override public Map generate(NameValue nameValue) { - initializeNecessaryFields(nameValue); + initializeNecessaryFields(Objects.requireNonNull(nameValue)); Map mapToExport = collectExportData(); return Map.of(DSM_OBJECT, mapToExport); } From 89613682919305cba65031ad793ffe774b3b7f3e Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 20 Oct 2021 11:32:36 +0400 Subject: [PATCH 039/405] DDP-7047 | introduce new classes for solving problem related to upserting mappings in ES --- .../dsm/model/elastic/export/BaseExporter.java | 11 ++++++++--- .../elastic/export/ElasticExportAdapter.java | 12 +++++++----- .../dsm/model/elastic/export/Exportable.java | 3 ++- .../model/elastic/export/MappingGenerator.java | 7 +++++++ .../model/elastic/export/SourceGenerator.java | 2 +- ...yload.java => UpsertDataRequestPayload.java} | 8 ++++---- .../export/UpsertMappingRequestPayload.java | 17 +++++++++++++++++ .../{Generator.java => ValueGenerator.java} | 2 +- .../dsm/model/patch/BasePatch.java | 14 +++++++------- ...rTest.java => SourceValueGeneratorTest.java} | 4 ++-- 10 files changed, 56 insertions(+), 24 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java rename src/main/java/org/broadinstitute/dsm/model/elastic/export/{UpdateRequestPayload.java => UpsertDataRequestPayload.java} (90%) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/UpsertMappingRequestPayload.java rename src/main/java/org/broadinstitute/dsm/model/elastic/export/{Generator.java => ValueGenerator.java} (84%) rename src/test/java/org/broadinstitute/dsm/model/elastic/export/{SourceGeneratorTest.java => SourceValueGeneratorTest.java} (93%) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java index dabc1c440..5ef714ba2 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java @@ -1,10 +1,15 @@ package org.broadinstitute.dsm.model.elastic.export; +import org.broadinstitute.dsm.util.ElasticSearchUtil; +import org.elasticsearch.client.RestHighLevelClient; + public abstract class BaseExporter implements Exportable { - protected UpdateRequestPayload updateRequestPayload; + protected RestHighLevelClient clientInstance = ElasticSearchUtil.getClientInstance(); + + protected UpsertDataRequestPayload upsertDataRequestPayload; - public void setUpdateRequestPayload(UpdateRequestPayload updateRequestPayload) { - this.updateRequestPayload = updateRequestPayload; + public void setUpdateRequestPayload(UpsertDataRequestPayload upsertDataRequestPayload) { + this.upsertDataRequestPayload = upsertDataRequestPayload; } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticExportAdapter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticExportAdapter.java index ab81e1836..c10595de9 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticExportAdapter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticExportAdapter.java @@ -1,10 +1,8 @@ package org.broadinstitute.dsm.model.elastic.export; -import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestHighLevelClient; import java.io.IOException; import java.util.Map; @@ -13,13 +11,17 @@ public class ElasticExportAdapter extends BaseExporter { @Override - public void export(Map data) { - RestHighLevelClient clientInstance = ElasticSearchUtil.getClientInstance(); - UpdateRequest updateRequest = Objects.requireNonNull(updateRequestPayload).getUpdateRequest(data); + public void exportData(Map data) { + UpdateRequest updateRequest = Objects.requireNonNull(upsertDataRequestPayload).getUpdateRequest(data); try { clientInstance.update(updateRequest, RequestOptions.DEFAULT); } catch (IOException e) { throw new RuntimeException("Error while exporting data to ES", e); } } + + @Override + public void exportMapping(Map mapping) { + + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java index c773311ca..9a7ff81c3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java @@ -4,6 +4,7 @@ public interface Exportable { - void export(Map data); + void exportData(Map data); + void exportMapping(Map mapping); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java new file mode 100644 index 000000000..c4d545871 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java @@ -0,0 +1,7 @@ +package org.broadinstitute.dsm.model.elastic.export; + +import java.util.Map; + +public interface MappingGenerator { + Map generate(); +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java index 5e047973b..0c8053d19 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java @@ -10,7 +10,7 @@ import java.util.Map; import java.util.Objects; -public class SourceGenerator implements Generator { +public class SourceGenerator implements ValueGenerator { public static final String DSM_OBJECT = "dsm"; public static final Map TABLE_ALIAS_MAPPINGS = Map.of( diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/UpdateRequestPayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/UpsertDataRequestPayload.java similarity index 90% rename from src/main/java/org/broadinstitute/dsm/model/elastic/export/UpdateRequestPayload.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/export/UpsertDataRequestPayload.java index 7463e0caf..d01212d86 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/UpdateRequestPayload.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/UpsertDataRequestPayload.java @@ -4,7 +4,7 @@ import java.util.Map; -public class UpdateRequestPayload { +public class UpsertDataRequestPayload { private static final String DOC = "_doc"; @@ -14,7 +14,7 @@ public class UpdateRequestPayload { private boolean docAsUpsert; private int retryOnConflict; - private UpdateRequestPayload(Builder builder) { + private UpsertDataRequestPayload(Builder builder) { this.index = builder.index; this.type = builder.type; this.id = builder.id; @@ -69,8 +69,8 @@ public Builder withRetryOnConflict(int retryOnConflict) { return this; } - public UpdateRequestPayload build() { - return new UpdateRequestPayload(this); + public UpsertDataRequestPayload build() { + return new UpsertDataRequestPayload(this); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/UpsertMappingRequestPayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/UpsertMappingRequestPayload.java new file mode 100644 index 000000000..8134f4e48 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/UpsertMappingRequestPayload.java @@ -0,0 +1,17 @@ +package org.broadinstitute.dsm.model.elastic.export; + +import org.elasticsearch.client.indices.PutMappingRequest; + +public class UpsertMappingRequestPayload { + + private String index; + + public UpsertMappingRequestPayload(String index) { + this.index = index; + } + + public PutMappingRequest getPutMappingRequest() { + return new PutMappingRequest(index); + } + +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Generator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ValueGenerator.java similarity index 84% rename from src/main/java/org/broadinstitute/dsm/model/elastic/export/Generator.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/export/ValueGenerator.java index 75c0907f3..8af9d6e2b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Generator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ValueGenerator.java @@ -4,6 +4,6 @@ import org.broadinstitute.dsm.model.NameValue; -public interface Generator { +public interface ValueGenerator { Map generate(NameValue nameValue); } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index 71d5bef6a..1f6edd2c8 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -24,9 +24,9 @@ import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.Value; import org.broadinstitute.dsm.model.elastic.ESProfile; -import org.broadinstitute.dsm.model.elastic.export.Generator; +import org.broadinstitute.dsm.model.elastic.export.ValueGenerator; import org.broadinstitute.dsm.model.elastic.export.BaseExporter; -import org.broadinstitute.dsm.model.elastic.export.UpdateRequestPayload; +import org.broadinstitute.dsm.model.elastic.export.UpsertDataRequestPayload; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; @@ -56,7 +56,7 @@ public abstract class BasePatch { protected DDPInstance ddpInstance; protected DBElement dbElement; protected BaseExporter exportable; - protected Generator generator; + protected ValueGenerator generator; { @@ -74,12 +74,12 @@ protected BasePatch(Patch patch) { protected void exportToES(NameValue nameValue) { Map elasticMapToExport = generator.generate(nameValue); - UpdateRequestPayload updateRequestPayload = new UpdateRequestPayload.Builder(ddpInstance.getParticipantIndexES(), patch.getDdpParticipantId()) + UpsertDataRequestPayload upsertDataRequestPayload = new UpsertDataRequestPayload.Builder(ddpInstance.getParticipantIndexES(), patch.getDdpParticipantId()) .withDocAsUpsert(true) .withRetryOnConflict(5) .build(); - exportable.setUpdateRequestPayload(updateRequestPayload); - exportable.export(elasticMapToExport); + exportable.setUpdateRequestPayload(upsertDataRequestPayload); + exportable.exportData(elasticMapToExport); } public abstract Object doPatch(); @@ -277,7 +277,7 @@ public void setExportable(BaseExporter exportable) { this.exportable = exportable; } - public void setGenerator(Generator generator) { + public void setGenerator(ValueGenerator generator) { this.generator = generator; } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceValueGeneratorTest.java similarity index 93% rename from src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java rename to src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceValueGeneratorTest.java index a65a1ff43..633e3c9ec 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceValueGeneratorTest.java @@ -8,13 +8,13 @@ import org.junit.Test; import org.junit.Assert; -public class SourceGeneratorTest { +public class SourceValueGeneratorTest { public static final String MEDICAL_RECORD_COLUMN = "medical_record_column"; @Test public void generate() { - Generator generator = new TestSourceGenerator(); + ValueGenerator generator = new TestSourceGenerator(); NameValue nameValue = new NameValue(MEDICAL_RECORD_COLUMN, "value"); Map objectMap = generator.generate(nameValue); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), SourceGenerator.DSM_OBJECT); From feb3aecee512d616802992346bedcce9a289ea77 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 20 Oct 2021 15:05:57 +0400 Subject: [PATCH 040/405] DDP-7047 | add mapping generator to generate mapper dynamically --- .../export/{ValueGenerator.java => Generator.java} | 2 +- .../dsm/model/elastic/export/MappingGenerator.java | 11 +++++++++-- .../dsm/model/elastic/export/SourceGenerator.java | 2 +- .../org/broadinstitute/dsm/model/patch/BasePatch.java | 6 +++--- ...lueGeneratorTest.java => SourceGeneratorTest.java} | 4 ++-- 5 files changed, 16 insertions(+), 9 deletions(-) rename src/main/java/org/broadinstitute/dsm/model/elastic/export/{ValueGenerator.java => Generator.java} (84%) rename src/test/java/org/broadinstitute/dsm/model/elastic/export/{SourceValueGeneratorTest.java => SourceGeneratorTest.java} (93%) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ValueGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/Generator.java similarity index 84% rename from src/main/java/org/broadinstitute/dsm/model/elastic/export/ValueGenerator.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/export/Generator.java index 8af9d6e2b..75c0907f3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ValueGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/Generator.java @@ -4,6 +4,6 @@ import org.broadinstitute.dsm.model.NameValue; -public interface ValueGenerator { +public interface Generator { Map generate(NameValue nameValue); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java index c4d545871..1aa23a5c9 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java @@ -2,6 +2,13 @@ import java.util.Map; -public interface MappingGenerator { - Map generate(); +import org.broadinstitute.dsm.model.NameValue; + +public class MappingGenerator implements Generator { + + + @Override + public Map generate(NameValue nameValue) { + return null; + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java index 0c8053d19..5e047973b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java @@ -10,7 +10,7 @@ import java.util.Map; import java.util.Objects; -public class SourceGenerator implements ValueGenerator { +public class SourceGenerator implements Generator { public static final String DSM_OBJECT = "dsm"; public static final Map TABLE_ALIAS_MAPPINGS = Map.of( diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index 1f6edd2c8..5b0623d35 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -24,7 +24,7 @@ import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.Value; import org.broadinstitute.dsm.model.elastic.ESProfile; -import org.broadinstitute.dsm.model.elastic.export.ValueGenerator; +import org.broadinstitute.dsm.model.elastic.export.Generator; import org.broadinstitute.dsm.model.elastic.export.BaseExporter; import org.broadinstitute.dsm.model.elastic.export.UpsertDataRequestPayload; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; @@ -56,7 +56,7 @@ public abstract class BasePatch { protected DDPInstance ddpInstance; protected DBElement dbElement; protected BaseExporter exportable; - protected ValueGenerator generator; + protected Generator generator; { @@ -277,7 +277,7 @@ public void setExportable(BaseExporter exportable) { this.exportable = exportable; } - public void setGenerator(ValueGenerator generator) { + public void setGenerator(Generator generator) { this.generator = generator; } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceValueGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java similarity index 93% rename from src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceValueGeneratorTest.java rename to src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java index 633e3c9ec..a65a1ff43 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceValueGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java @@ -8,13 +8,13 @@ import org.junit.Test; import org.junit.Assert; -public class SourceValueGeneratorTest { +public class SourceGeneratorTest { public static final String MEDICAL_RECORD_COLUMN = "medical_record_column"; @Test public void generate() { - ValueGenerator generator = new TestSourceGenerator(); + Generator generator = new TestSourceGenerator(); NameValue nameValue = new NameValue(MEDICAL_RECORD_COLUMN, "value"); Map objectMap = generator.generate(nameValue); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), SourceGenerator.DSM_OBJECT); From e4eaeaee20757e12dd1212612ff15ccc2f526746 Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 20 Oct 2021 15:38:45 +0400 Subject: [PATCH 041/405] DDP-7047 | introduce BaseGenerator which will separate Mapping and Source generators --- .../model/elastic/export/BaseGenerator.java | 34 +++++++++++++++++++ .../elastic/export/MappingGenerator.java | 28 ++++++++++++++- .../model/elastic/export/SourceGenerator.java | 24 +------------ .../dsm/model/patch/PatchFactory.java | 2 ++ 4 files changed, 64 insertions(+), 24 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java new file mode 100644 index 000000000..8b5becaa9 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java @@ -0,0 +1,34 @@ +package org.broadinstitute.dsm.model.elastic.export; + +import org.broadinstitute.dsm.db.structure.DBElement; +import org.broadinstitute.dsm.model.NameValue; +import org.broadinstitute.dsm.util.PatchUtil; + +import java.util.Map; +import java.util.Objects; + +public abstract class BaseGenerator implements Generator { + + + public static final String DSM_OBJECT = "dsm"; + public static final Map TABLE_ALIAS_MAPPINGS = Map.of( + "m", "medicalRecords", + "t", "tissueRecords", + "oD", "oncHistoryDetailRecords", + "r", "participant", + "p", "participant", + "d", "participant" + ); + + protected NameValue nameValue; + protected DBElement dbElement; + + protected void initializeNecessaryFields(NameValue nameValue) { + this.nameValue = nameValue; + dbElement = getDBElement(); + } + + protected DBElement getDBElement() { + return PatchUtil.getColumnNameMap().get(Objects.requireNonNull(nameValue).getName()); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java index 1aa23a5c9..af502dd0e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java @@ -1,14 +1,40 @@ package org.broadinstitute.dsm.model.elastic.export; +import java.util.HashMap; import java.util.Map; +import java.util.Objects; +import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.NameValue; -public class MappingGenerator implements Generator { +public class MappingGenerator extends BaseGenerator { @Override public Map generate(NameValue nameValue) { + initializeNecessaryFields(Objects.requireNonNull(nameValue)); + String property = TABLE_ALIAS_MAPPINGS.get(dbElement.getTableAlias()); + String value = (String) nameValue.getValue(); + + StringUtils.isNumeric("asd"); + Boolean.valueOf("true"); + // Date, time, datetime + + + + Map jsonMap = new HashMap<>(); + Map message = new HashMap<>(); + message.put("type", ""); + Map properties = new HashMap<>(); + properties.put("message", message); + jsonMap.put("properties", properties); + request.source(jsonMap); + + + return null; } + + + } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java index 5e047973b..8d507a688 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java @@ -2,27 +2,13 @@ import com.google.gson.Gson; import com.google.gson.JsonSyntaxException; -import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; -import org.broadinstitute.dsm.util.PatchUtil; import java.util.HashMap; import java.util.Map; import java.util.Objects; -public class SourceGenerator implements Generator { - - public static final String DSM_OBJECT = "dsm"; - public static final Map TABLE_ALIAS_MAPPINGS = Map.of( - "m", "medicalRecords", - "t", "tissueRecords", - "oD", "oncHistoryDetailRecords", - "r", "participant", - "p", "participant", - "d", "participant" - ); - protected NameValue nameValue; - protected DBElement dbElement; +public class SourceGenerator extends BaseGenerator { @Override public Map generate(NameValue nameValue) { @@ -31,11 +17,6 @@ public Map generate(NameValue nameValue) { return Map.of(DSM_OBJECT, mapToExport); } - protected void initializeNecessaryFields(NameValue nameValue) { - this.nameValue = nameValue; - dbElement = getDBElement(); - } - protected Map collectExportData() { Map result = new HashMap<>(); String objectKey = TABLE_ALIAS_MAPPINGS.get(dbElement.getTableAlias()); @@ -49,7 +30,4 @@ protected Map collectExportData() { return result; } - protected DBElement getDBElement() { - return PatchUtil.getColumnNameMap().get(Objects.requireNonNull(nameValue).getName()); - } } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java index 1586d008a..2b29b0cc3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java @@ -2,6 +2,7 @@ import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.elastic.export.ElasticExportAdapter; +import org.broadinstitute.dsm.model.elastic.export.MappingGenerator; import org.broadinstitute.dsm.model.elastic.export.SourceGenerator; import org.broadinstitute.dsm.util.NotificationUtil; @@ -28,6 +29,7 @@ public static BasePatch makePatch(Patch patch, NotificationUtil notificationUtil } patcher.setExportable(new ElasticExportAdapter()); patcher.setGenerator(new SourceGenerator()); + patcher.setGenerator(new MappingGenerator()); if (patcher instanceof NullPatch) { throw new RuntimeException("Id and parentId was null"); } From b5afc8532996246bcb52fdbe7bb527bb5a0af26b Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 20 Oct 2021 16:23:35 +0400 Subject: [PATCH 042/405] DDP-7047 | start implementing logic for parses for elastic mapper --- .../elastic/export/MappingGenerator.java | 37 +++++++++++++++---- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java index af502dd0e..b676e73e1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java @@ -1,8 +1,13 @@ package org.broadinstitute.dsm.model.elastic.export; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.format.DateTimeParseException; import java.util.HashMap; import java.util.Map; import java.util.Objects; +import java.util.function.Predicate; import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.NameValue; @@ -15,26 +20,44 @@ public Map generate(NameValue nameValue) { initializeNecessaryFields(Objects.requireNonNull(nameValue)); String property = TABLE_ALIAS_MAPPINGS.get(dbElement.getTableAlias()); String value = (String) nameValue.getValue(); + String type = "text"; - StringUtils.isNumeric("asd"); - Boolean.valueOf("true"); - // Date, time, datetime - + if (StringUtils.isNumeric(value)) { + type = "integer"; + } else if (isBoolean(value)) { + type = "boolean"; + } else if (isDateTime(value)) { + type = "date"; + } + //properties.dsm.properties.medicalRecords -> {"type": "boolean"} Map jsonMap = new HashMap<>(); Map message = new HashMap<>(); - message.put("type", ""); + message.put("type", type); Map properties = new HashMap<>(); properties.put("message", message); jsonMap.put("properties", properties); - request.source(jsonMap); - + // Date, time, datetime return null; } + private boolean isDateTime(String value) { + try { + LocalDate.parse(value); + LocalTime.parse(value); + LocalDateTime.parse(value); + } catch (DateTimeParseException dtpe) { + return false; + } + return true; + } + + private boolean isBoolean(String value) { + return value.equalsIgnoreCase("true") || value.equalsIgnoreCase("false"); + } } From 1221c7473c22c8d9aec5d383c7ca5d77ea43f978 Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 20 Oct 2021 16:57:49 +0400 Subject: [PATCH 043/405] DDp-7047 | initialize tests for MappingGenerator --- .../model/elastic/export/BaseGenerator.java | 1 + .../elastic/export/MappingGenerator.java | 18 ++++------ .../elastic/export/MappingGeneratorTest.java | 33 +++++++++++++++++++ .../elastic/export/SourceGeneratorTest.java | 15 ++------- .../model/elastic/export/TestPatchUtil.java | 17 ++++++++++ 5 files changed, 59 insertions(+), 25 deletions(-) create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/export/TestPatchUtil.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java index 8b5becaa9..529c3dcb0 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java @@ -11,6 +11,7 @@ public abstract class BaseGenerator implements Generator { public static final String DSM_OBJECT = "dsm"; + public static final String PROPERTIES = "properties"; public static final Map TABLE_ALIAS_MAPPINGS = Map.of( "m", "medicalRecords", "t", "tissueRecords", diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java index b676e73e1..2f4311d9b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java @@ -7,7 +7,6 @@ import java.util.HashMap; import java.util.Map; import java.util.Objects; -import java.util.function.Predicate; import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.NameValue; @@ -31,17 +30,12 @@ public Map generate(NameValue nameValue) { type = "date"; } - //properties.dsm.properties.medicalRecords -> {"type": "boolean"} - Map jsonMap = new HashMap<>(); - Map message = new HashMap<>(); - message.put("type", type); - Map properties = new HashMap<>(); - properties.put("message", message); - jsonMap.put("properties", properties); - // Date, time, datetime - - - return null; + Map fieldType = Map.of("type", type); + Map>> mappedField = Map.of(PROPERTIES, Map.of(dbElement.getColumnName(), fieldType)); + Map>>> dsmLevelProperty = Map.of(property, mappedField); + Map dsmLevelProperties = Map.of(PROPERTIES, dsmLevelProperty); + Map> dsmLevel = Map.of(DSM_OBJECT, dsmLevelProperties); + return Map.of(PROPERTIES, dsmLevel); } private boolean isDateTime(String value) { diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java new file mode 100644 index 000000000..19d82bf11 --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java @@ -0,0 +1,33 @@ +package org.broadinstitute.dsm.model.elastic.export; + + +import org.broadinstitute.dsm.db.structure.DBElement; +import org.broadinstitute.dsm.model.NameValue; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Map; + +public class MappingGeneratorTest { + + @Test + public void generate() { + Generator generator = new TestMappingGenerator(); + NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "value"); + Map objectMap = generator.generate(nameValue); + Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), BaseGenerator.PROPERTIES); + ((Map) ((Map)((Map) ((Map)((Map) objectMap.get(BaseGenerator.PROPERTIES)).get(BaseGenerator.DSM_OBJECT)).get(BaseGenerator.PROPERTIES)).get("medicalRecords")).get(BaseGenerator.PROPERTIES)).get(); + String value = (String)((Map)((Map)objectMap.get(SourceGenerator.DSM_OBJECT)).get("medicalRecords")).get(TestPatchUtil.MEDICAL_RECORD_COLUMN); + Assert.assertEquals("value", value); + + + } + + private static class TestMappingGenerator extends MappingGenerator { + + @Override + protected DBElement getDBElement() { + return TestPatchUtil.getColumnNameMap().get(nameValue.getName()); + } + } +} \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java index a65a1ff43..aecc05c86 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java @@ -4,21 +4,18 @@ import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; -import org.broadinstitute.dsm.util.PatchUtil; import org.junit.Test; import org.junit.Assert; public class SourceGeneratorTest { - public static final String MEDICAL_RECORD_COLUMN = "medical_record_column"; - @Test public void generate() { Generator generator = new TestSourceGenerator(); - NameValue nameValue = new NameValue(MEDICAL_RECORD_COLUMN, "value"); + NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "value"); Map objectMap = generator.generate(nameValue); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), SourceGenerator.DSM_OBJECT); - String value = (String)((Map)((Map)objectMap.get(SourceGenerator.DSM_OBJECT)).get("medicalRecords")).get(MEDICAL_RECORD_COLUMN); + String value = (String)((Map)((Map)objectMap.get(SourceGenerator.DSM_OBJECT)).get("medicalRecords")).get(TestPatchUtil.MEDICAL_RECORD_COLUMN); Assert.assertEquals("value", value); } @@ -30,13 +27,5 @@ protected DBElement getDBElement() { } } - private static class TestPatchUtil extends PatchUtil { - - public static Map getColumnNameMap() { - DBElement dbElement = new DBElement("ddp_medical_record", "m", "pr", MEDICAL_RECORD_COLUMN); - return Map.of(MEDICAL_RECORD_COLUMN, dbElement); - } - - } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/TestPatchUtil.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/TestPatchUtil.java new file mode 100644 index 000000000..66b18d772 --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/TestPatchUtil.java @@ -0,0 +1,17 @@ +package org.broadinstitute.dsm.model.elastic.export; + +import org.broadinstitute.dsm.db.structure.DBElement; +import org.broadinstitute.dsm.util.PatchUtil; + +import java.util.Map; + +class TestPatchUtil extends PatchUtil { + + public static final String MEDICAL_RECORD_COLUMN = "medical_record_column"; + + public static Map getColumnNameMap() { + DBElement dbElement = new DBElement("ddp_medical_record", "m", "pr", MEDICAL_RECORD_COLUMN); + return Map.of(MEDICAL_RECORD_COLUMN, dbElement); + } + +} From fb5ad23520dc9f46eaf807a893054273c24c071d Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 20 Oct 2021 17:22:22 +0400 Subject: [PATCH 044/405] DDP-7047 | add tests for mapping generator --- .../elastic/export/MappingGenerator.java | 27 ++++++++-- .../elastic/export/MappingGeneratorTest.java | 52 +++++++++++++++++-- 2 files changed, 71 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java index 2f4311d9b..952e52a6d 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java @@ -4,7 +4,6 @@ import java.time.LocalDateTime; import java.time.LocalTime; import java.time.format.DateTimeParseException; -import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -26,7 +25,7 @@ public Map generate(NameValue nameValue) { type = "integer"; } else if (isBoolean(value)) { type = "boolean"; - } else if (isDateTime(value)) { + } else if (isDateOrTimeOrDateTime(value)) { type = "date"; } @@ -38,10 +37,12 @@ public Map generate(NameValue nameValue) { return Map.of(PROPERTIES, dsmLevel); } + private boolean isDateOrTimeOrDateTime(String value) { + return isDate(value) || isTime(value) || isDateTime(value); + } + private boolean isDateTime(String value) { try { - LocalDate.parse(value); - LocalTime.parse(value); LocalDateTime.parse(value); } catch (DateTimeParseException dtpe) { return false; @@ -49,6 +50,24 @@ private boolean isDateTime(String value) { return true; } + private boolean isTime(String value) { + try { + LocalTime.parse(value); + } catch (DateTimeParseException dtpe) { + return false; + } + return true; + } + + private boolean isDate(String value) { + try { + LocalDate.parse(value); + } catch (DateTimeParseException dtpe) { + return false; + } + return true; + } + private boolean isBoolean(String value) { return value.equalsIgnoreCase("true") || value.equalsIgnoreCase("false"); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java index 19d82bf11..ff80341c8 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java @@ -11,16 +11,60 @@ public class MappingGeneratorTest { @Test - public void generate() { + public void generateTextType() { Generator generator = new TestMappingGenerator(); NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "value"); Map objectMap = generator.generate(nameValue); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), BaseGenerator.PROPERTIES); - ((Map) ((Map)((Map) ((Map)((Map) objectMap.get(BaseGenerator.PROPERTIES)).get(BaseGenerator.DSM_OBJECT)).get(BaseGenerator.PROPERTIES)).get("medicalRecords")).get(BaseGenerator.PROPERTIES)).get(); - String value = (String)((Map)((Map)objectMap.get(SourceGenerator.DSM_OBJECT)).get("medicalRecords")).get(TestPatchUtil.MEDICAL_RECORD_COLUMN); - Assert.assertEquals("value", value); + String type = extractDeepestLeveleValue(objectMap); + Assert.assertEquals("text", type); + } + + @Test + public void generateBooleanType() { + Generator generator = new TestMappingGenerator(); + NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "true"); + Map objectMap = generator.generate(nameValue); + Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), BaseGenerator.PROPERTIES); + String type = extractDeepestLeveleValue(objectMap); + Assert.assertEquals("boolean", type); + } + + @Test + public void generateIntegerType() { + Generator generator = new TestMappingGenerator(); + NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "45"); + Map objectMap = generator.generate(nameValue); + Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), BaseGenerator.PROPERTIES); + String type = extractDeepestLeveleValue(objectMap); + Assert.assertEquals("integer", type); + } + @Test + public void generateDateType() { + Generator generator = new TestMappingGenerator(); + NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "2021-10-30"); + Map objectMap = generator.generate(nameValue); + Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), BaseGenerator.PROPERTIES); + String type = extractDeepestLeveleValue(objectMap); + Assert.assertEquals("date", type); + } + private String extractDeepestLeveleValue(Map objectMap) { + return (String) + ((Map) + ((Map) + ((Map) + ((Map) + ((Map) + ((Map) objectMap + .get(BaseGenerator.PROPERTIES)) + .get(BaseGenerator.DSM_OBJECT)) + .get(BaseGenerator.PROPERTIES)) + .get("medicalRecords")) + .get(BaseGenerator.PROPERTIES)) + .get("medical_record_column")) + .get("type"); } private static class TestMappingGenerator extends MappingGenerator { From d5fc8a567081e314fa5f0a322b8d03d3346e5445 Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 20 Oct 2021 18:15:23 +0400 Subject: [PATCH 045/405] DDP-7047 | further modifications --- .../model/elastic/export/BaseExporter.java | 10 ++- .../elastic/export/ElasticExportAdapter.java | 12 +++- .../model/elastic/export/ExportFacade.java | 16 +++++ .../elastic/export/MappingGenerator.java | 61 ++--------------- .../dsm/model/elastic/export/Parser.java | 7 ++ .../dsm/model/elastic/export/TypeParser.java | 65 +++++++++++++++++++ .../dsm/model/patch/BasePatch.java | 7 +- .../dsm/model/patch/PatchFactory.java | 3 +- .../elastic/export/MappingGeneratorTest.java | 16 +++-- 9 files changed, 129 insertions(+), 68 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/Parser.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/TypeParser.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java index 5ef714ba2..4b0e0e5c5 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java @@ -3,13 +3,19 @@ import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.elasticsearch.client.RestHighLevelClient; +import java.util.Objects; + public abstract class BaseExporter implements Exportable { protected RestHighLevelClient clientInstance = ElasticSearchUtil.getClientInstance(); - protected UpsertDataRequestPayload upsertDataRequestPayload; + protected UpsertMappingRequestPayload upsertMappingRequestPayload; public void setUpdateRequestPayload(UpsertDataRequestPayload upsertDataRequestPayload) { - this.upsertDataRequestPayload = upsertDataRequestPayload; + this.upsertDataRequestPayload = Objects.requireNonNull(upsertDataRequestPayload); + } + + public void setUpsertMappingRequestPayload(UpsertMappingRequestPayload upsertMappingRequestPayload) { + this.upsertMappingRequestPayload = Objects.requireNonNull(upsertMappingRequestPayload); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticExportAdapter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticExportAdapter.java index c10595de9..824250583 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticExportAdapter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticExportAdapter.java @@ -3,16 +3,16 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.indices.PutMappingRequest; import java.io.IOException; import java.util.Map; -import java.util.Objects; public class ElasticExportAdapter extends BaseExporter { @Override public void exportData(Map data) { - UpdateRequest updateRequest = Objects.requireNonNull(upsertDataRequestPayload).getUpdateRequest(data); + UpdateRequest updateRequest = upsertDataRequestPayload.getUpdateRequest(data); try { clientInstance.update(updateRequest, RequestOptions.DEFAULT); } catch (IOException e) { @@ -22,6 +22,12 @@ public void exportData(Map data) { @Override public void exportMapping(Map mapping) { - + PutMappingRequest putMappingRequest = upsertMappingRequestPayload.getPutMappingRequest(); + putMappingRequest.source(mapping); + try { + clientInstance.indices().putMapping(putMappingRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + throw new RuntimeException("Error while updating mapping to ES", e); + } } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java new file mode 100644 index 000000000..98fad7b97 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -0,0 +1,16 @@ +package org.broadinstitute.dsm.model.elastic.export; + +public class ExportFacade { + + Exportable exportable; + Generator generator; + Parser parser; + + public ExportFacade(Exportable exportable, Generator generator, Parser parser) { + this.exportable = exportable; + this.generator = generator; + this.parser = parser; + } + + +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java index 952e52a6d..d9a9cf03b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java @@ -1,34 +1,24 @@ package org.broadinstitute.dsm.model.elastic.export; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.LocalTime; -import java.time.format.DateTimeParseException; import java.util.Map; import java.util.Objects; -import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.NameValue; public class MappingGenerator extends BaseGenerator { + private final Parser typeParser; + + public MappingGenerator(Parser typeParser) { + this.typeParser = typeParser; + } + @Override public Map generate(NameValue nameValue) { initializeNecessaryFields(Objects.requireNonNull(nameValue)); String property = TABLE_ALIAS_MAPPINGS.get(dbElement.getTableAlias()); - String value = (String) nameValue.getValue(); - String type = "text"; - - - if (StringUtils.isNumeric(value)) { - type = "integer"; - } else if (isBoolean(value)) { - type = "boolean"; - } else if (isDateOrTimeOrDateTime(value)) { - type = "date"; - } - + String type = typeParser.parseType(nameValue); Map fieldType = Map.of("type", type); Map>> mappedField = Map.of(PROPERTIES, Map.of(dbElement.getColumnName(), fieldType)); Map>>> dsmLevelProperty = Map.of(property, mappedField); @@ -36,41 +26,4 @@ public Map generate(NameValue nameValue) { Map> dsmLevel = Map.of(DSM_OBJECT, dsmLevelProperties); return Map.of(PROPERTIES, dsmLevel); } - - private boolean isDateOrTimeOrDateTime(String value) { - return isDate(value) || isTime(value) || isDateTime(value); - } - - private boolean isDateTime(String value) { - try { - LocalDateTime.parse(value); - } catch (DateTimeParseException dtpe) { - return false; - } - return true; - } - - private boolean isTime(String value) { - try { - LocalTime.parse(value); - } catch (DateTimeParseException dtpe) { - return false; - } - return true; - } - - private boolean isDate(String value) { - try { - LocalDate.parse(value); - } catch (DateTimeParseException dtpe) { - return false; - } - return true; - } - - private boolean isBoolean(String value) { - return value.equalsIgnoreCase("true") || value.equalsIgnoreCase("false"); - } - - } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Parser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/Parser.java new file mode 100644 index 000000000..a7b237a1a --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/Parser.java @@ -0,0 +1,7 @@ +package org.broadinstitute.dsm.model.elastic.export; + +import org.broadinstitute.dsm.model.NameValue; + +public interface Parser { + String parseType(NameValue nameValue); +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/TypeParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/TypeParser.java new file mode 100644 index 000000000..1a3d459c9 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/TypeParser.java @@ -0,0 +1,65 @@ +package org.broadinstitute.dsm.model.elastic.export; + +import org.apache.commons.lang3.StringUtils; +import org.broadinstitute.dsm.model.NameValue; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.format.DateTimeParseException; + +public class TypeParser implements Parser { + public TypeParser() { + } + + @Override + public String parseType(NameValue nameValue) { + String value = (String) nameValue.getValue(); + String type = "text"; + + + if (StringUtils.isNumeric(value)) { + type = "integer"; + } else if (isBoolean(value)) { + type = "boolean"; + } else if (isDateOrTimeOrDateTime(value)) { + type = "date"; + } + return type; + } + + boolean isDateOrTimeOrDateTime(String value) { + return isDate(value) || isTime(value) || isDateTime(value); + } + + boolean isDateTime(String value) { + try { + LocalDateTime.parse(value); + } catch (DateTimeParseException dtpe) { + return false; + } + return true; + } + + boolean isTime(String value) { + try { + LocalTime.parse(value); + } catch (DateTimeParseException dtpe) { + return false; + } + return true; + } + + boolean isDate(String value) { + try { + LocalDate.parse(value); + } catch (DateTimeParseException dtpe) { + return false; + } + return true; + } + + boolean isBoolean(String value) { + return value.equalsIgnoreCase("true") || value.equalsIgnoreCase("false"); + } +} \ No newline at end of file diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index 5b0623d35..ef3b4f640 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -24,9 +24,7 @@ import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.Value; import org.broadinstitute.dsm.model.elastic.ESProfile; -import org.broadinstitute.dsm.model.elastic.export.Generator; -import org.broadinstitute.dsm.model.elastic.export.BaseExporter; -import org.broadinstitute.dsm.model.elastic.export.UpsertDataRequestPayload; +import org.broadinstitute.dsm.model.elastic.export.*; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; @@ -78,6 +76,9 @@ protected void exportToES(NameValue nameValue) { .withDocAsUpsert(true) .withRetryOnConflict(5) .build(); + UpsertMappingRequestPayload upsertMappingRequestPayload = new UpsertMappingRequestPayload(ddpInstance.getParticipantIndexES()); + exportable.setUpsertMappingRequestPayload(upsertMappingRequestPayload); + exportable.exportMapping(); exportable.setUpdateRequestPayload(upsertDataRequestPayload); exportable.exportData(elasticMapToExport); } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java index 2b29b0cc3..e05e0f5a2 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java @@ -4,6 +4,7 @@ import org.broadinstitute.dsm.model.elastic.export.ElasticExportAdapter; import org.broadinstitute.dsm.model.elastic.export.MappingGenerator; import org.broadinstitute.dsm.model.elastic.export.SourceGenerator; +import org.broadinstitute.dsm.model.elastic.export.TypeParser; import org.broadinstitute.dsm.util.NotificationUtil; public class PatchFactory { @@ -28,8 +29,6 @@ public static BasePatch makePatch(Patch patch, NotificationUtil notificationUtil } } patcher.setExportable(new ElasticExportAdapter()); - patcher.setGenerator(new SourceGenerator()); - patcher.setGenerator(new MappingGenerator()); if (patcher instanceof NullPatch) { throw new RuntimeException("Id and parentId was null"); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java index ff80341c8..98128fb2d 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java @@ -4,15 +4,22 @@ import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; import org.junit.Assert; +import org.junit.BeforeClass; import org.junit.Test; import java.util.Map; public class MappingGeneratorTest { + static Generator generator; + + @BeforeClass + public static void setUp() { + generator = new TestMappingGenerator(new TypeParser()); + } + @Test public void generateTextType() { - Generator generator = new TestMappingGenerator(); NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "value"); Map objectMap = generator.generate(nameValue); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), BaseGenerator.PROPERTIES); @@ -22,7 +29,6 @@ public void generateTextType() { @Test public void generateBooleanType() { - Generator generator = new TestMappingGenerator(); NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "true"); Map objectMap = generator.generate(nameValue); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), BaseGenerator.PROPERTIES); @@ -32,7 +38,6 @@ public void generateBooleanType() { @Test public void generateIntegerType() { - Generator generator = new TestMappingGenerator(); NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "45"); Map objectMap = generator.generate(nameValue); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), BaseGenerator.PROPERTIES); @@ -42,7 +47,6 @@ public void generateIntegerType() { @Test public void generateDateType() { - Generator generator = new TestMappingGenerator(); NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "2021-10-30"); Map objectMap = generator.generate(nameValue); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), BaseGenerator.PROPERTIES); @@ -69,6 +73,10 @@ private String extractDeepestLeveleValue(Map objectMap) { private static class TestMappingGenerator extends MappingGenerator { + public TestMappingGenerator(Parser typeParser) { + super(typeParser); + } + @Override protected DBElement getDBElement() { return TestPatchUtil.getColumnNameMap().get(nameValue.getName()); From 212b6e1c42e8df45611a3c2428e063b3f8295d96 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 20 Oct 2021 18:48:07 +0400 Subject: [PATCH 046/405] DDP-7047 | add facade class for elastic export --- .../model/elastic/export/ExportFacade.java | 40 +++++++++++++++---- .../elastic/export/ExportFacadePayload.java | 29 ++++++++++++++ .../dsm/model/patch/BasePatch.java | 23 ++--------- .../dsm/model/patch/PatchFactory.java | 1 - 4 files changed, 66 insertions(+), 27 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 98fad7b97..9eb988cce 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -1,16 +1,42 @@ package org.broadinstitute.dsm.model.elastic.export; +import java.util.Map; + public class ExportFacade { - Exportable exportable; + BaseExporter exportable; Generator generator; - Parser parser; + private ExportFacadePayload exportFacadePayload; + + public ExportFacade(ExportFacadePayload exportFacadePayload) { + this.exportFacadePayload = exportFacadePayload; + exportable = new ElasticExportAdapter(); + generator = new MappingGenerator(new TypeParser()); + } + + public void export() { + upsertMapping(); + upsertData(); + } + + private void upsertMapping() { + Map mappingToUpsert = generator.generate(exportFacadePayload.getNameValue()); + UpsertMappingRequestPayload upsertMappingRequestPayload = new UpsertMappingRequestPayload(exportFacadePayload.getIndex()); + exportable.setUpsertMappingRequestPayload(upsertMappingRequestPayload); + exportable.exportMapping(mappingToUpsert); + } - public ExportFacade(Exportable exportable, Generator generator, Parser parser) { - this.exportable = exportable; - this.generator = generator; - this.parser = parser; + private void upsertData() { + generator = new SourceGenerator(); + Map elasticDataToExport = generator.generate(exportFacadePayload.getNameValue()); + UpsertDataRequestPayload upsertDataRequestPayload = new UpsertDataRequestPayload.Builder(exportFacadePayload.getIndex(), + exportFacadePayload.getId()) + .withDocAsUpsert(true) + .withRetryOnConflict(5) + .build(); + exportable.setUpdateRequestPayload(upsertDataRequestPayload); + exportable.exportData(elasticDataToExport); } - + } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java new file mode 100644 index 000000000..c404a7749 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java @@ -0,0 +1,29 @@ +package org.broadinstitute.dsm.model.elastic.export; + +import java.util.Objects; + +import org.broadinstitute.dsm.model.NameValue; + +public class ExportFacadePayload { + private String index; + private String id; + private NameValue nameValue; + + public ExportFacadePayload(String index, String id, NameValue nameValue) { + this.index = Objects.requireNonNull(index); + this.id = Objects.requireNonNull(id); + this.nameValue = Objects.requireNonNull(nameValue); + } + + public String getIndex() { + return index; + } + + public String getId() { + return id; + } + + public NameValue getNameValue() { + return nameValue; + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index ef3b4f640..828e2664a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -53,8 +53,6 @@ public abstract class BasePatch { protected ESProfile profile; protected DDPInstance ddpInstance; protected DBElement dbElement; - protected BaseExporter exportable; - protected Generator generator; { @@ -71,16 +69,10 @@ protected BasePatch(Patch patch) { } protected void exportToES(NameValue nameValue) { - Map elasticMapToExport = generator.generate(nameValue); - UpsertDataRequestPayload upsertDataRequestPayload = new UpsertDataRequestPayload.Builder(ddpInstance.getParticipantIndexES(), patch.getDdpParticipantId()) - .withDocAsUpsert(true) - .withRetryOnConflict(5) - .build(); - UpsertMappingRequestPayload upsertMappingRequestPayload = new UpsertMappingRequestPayload(ddpInstance.getParticipantIndexES()); - exportable.setUpsertMappingRequestPayload(upsertMappingRequestPayload); - exportable.exportMapping(); - exportable.setUpdateRequestPayload(upsertDataRequestPayload); - exportable.exportData(elasticMapToExport); + ExportFacadePayload exportFacadePayload = + new ExportFacadePayload(ddpInstance.getParticipantIndexES(), patch.getDdpParticipantId(), nameValue); + ExportFacade exportFacade = new ExportFacade(exportFacadePayload); + exportFacade.export(); } public abstract Object doPatch(); @@ -274,11 +266,4 @@ private NameValue setAdditionalValue(String additionalValue, @NonNull Patch patc } } - public void setExportable(BaseExporter exportable) { - this.exportable = exportable; - } - - public void setGenerator(Generator generator) { - this.generator = generator; } - } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java index e05e0f5a2..69bca6081 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java @@ -28,7 +28,6 @@ public static BasePatch makePatch(Patch patch, NotificationUtil notificationUtil patcher = new ParticipantRecordPatch(patch); } } - patcher.setExportable(new ElasticExportAdapter()); if (patcher instanceof NullPatch) { throw new RuntimeException("Id and parentId was null"); } From bcd533a09f0cfcb7e7e453d29564a9417fc535ae Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 21 Oct 2021 10:45:08 +0400 Subject: [PATCH 047/405] DDP-7047 | changes according to elasticsearch --- .../dsm/model/elastic/export/BaseParser.java | 67 +++++++++++++++++++ .../model/elastic/export/ExportFacade.java | 3 +- .../elastic/export/MappingGenerator.java | 32 ++++++--- .../dsm/model/elastic/export/Parser.java | 3 +- .../model/elastic/export/SourceGenerator.java | 12 +++- .../dsm/model/elastic/export/TypeParser.java | 59 ++++------------ .../dsm/model/elastic/export/ValueParser.java | 24 +++++++ .../elastic/export/SourceGeneratorTest.java | 6 +- 8 files changed, 145 insertions(+), 61 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseParser.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/ValueParser.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseParser.java new file mode 100644 index 000000000..bd32f6e21 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseParser.java @@ -0,0 +1,67 @@ +package org.broadinstitute.dsm.model.elastic.export; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.format.DateTimeParseException; + +import org.apache.commons.lang3.StringUtils; + +public abstract class BaseParser implements Parser { + + @Override + public Object parse(String value) { + Object result = forString(value); + if (StringUtils.isNumeric(value)) { + result = forNumeric(value); + } else if (isBoolean(value)) { + result = forBoolean(value); + } else if (isDateOrTimeOrDateTime(value)) { + result = forDate(value); + } + return result; + } + + protected abstract Object forNumeric(String value); + + protected abstract Object forBoolean(String value); + + protected abstract Object forDate(String value); + + protected abstract Object forString(String value); + + boolean isDateOrTimeOrDateTime(String value) { + return isDate(value) || isTime(value) || isDateTime(value); + } + + boolean isDateTime(String value) { + try { + LocalDateTime.parse(value); + } catch (DateTimeParseException dtpe) { + return false; + } + return true; + } + + boolean isTime(String value) { + try { + LocalTime.parse(value); + } catch (DateTimeParseException dtpe) { + return false; + } + return true; + } + + boolean isDate(String value) { + try { + LocalDate.parse(value); + } catch (DateTimeParseException dtpe) { + return false; + } + return true; + } + + boolean isBoolean(String value) { + return value.equalsIgnoreCase("true") || value.equalsIgnoreCase("false"); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 9eb988cce..fc7622595 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -20,6 +20,7 @@ public void export() { } private void upsertMapping() { + generator = new MappingGenerator(new TypeParser()); Map mappingToUpsert = generator.generate(exportFacadePayload.getNameValue()); UpsertMappingRequestPayload upsertMappingRequestPayload = new UpsertMappingRequestPayload(exportFacadePayload.getIndex()); exportable.setUpsertMappingRequestPayload(upsertMappingRequestPayload); @@ -27,7 +28,7 @@ private void upsertMapping() { } private void upsertData() { - generator = new SourceGenerator(); + generator = new SourceGenerator(new ValueParser()); Map elasticDataToExport = generator.generate(exportFacadePayload.getNameValue()); UpsertDataRequestPayload upsertDataRequestPayload = new UpsertDataRequestPayload.Builder(exportFacadePayload.getIndex(), exportFacadePayload.getId()) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java index d9a9cf03b..942750514 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java @@ -1,29 +1,41 @@ package org.broadinstitute.dsm.model.elastic.export; +import java.util.HashMap; import java.util.Map; import java.util.Objects; +import com.google.gson.Gson; +import com.google.gson.JsonSyntaxException; import org.broadinstitute.dsm.model.NameValue; public class MappingGenerator extends BaseGenerator { - private final Parser typeParser; + private final Parser parser; - public MappingGenerator(Parser typeParser) { - this.typeParser = typeParser; + public MappingGenerator(Parser parser) { + this.parser = parser; } @Override public Map generate(NameValue nameValue) { initializeNecessaryFields(Objects.requireNonNull(nameValue)); String property = TABLE_ALIAS_MAPPINGS.get(dbElement.getTableAlias()); - String type = typeParser.parseType(nameValue); - Map fieldType = Map.of("type", type); - Map>> mappedField = Map.of(PROPERTIES, Map.of(dbElement.getColumnName(), fieldType)); - Map>>> dsmLevelProperty = Map.of(property, mappedField); - Map dsmLevelProperties = Map.of(PROPERTIES, dsmLevelProperty); - Map> dsmLevel = Map.of(DSM_OBJECT, dsmLevelProperties); - return Map.of(PROPERTIES, dsmLevel); + Map> field = new HashMap<>(); + try { + Map fieldsByValues = new Gson().fromJson((String) nameValue.getValue(), Map.class); + for (Map.Entry entry: fieldsByValues.entrySet()) { + Object eachType = parser.parse((String) entry.getValue()); + field.put(entry.getKey(), Map.of("type", eachType)); + } + } catch (JsonSyntaxException jse) { + Object type = parser.parse((String)nameValue.getValue()); + field = Map.of(dbElement.getColumnName(), Map.of("type", type)); // ddp_medical_column : {type: "text"} + } + Map>> mappedField = Map.of(PROPERTIES, field); //properties + Map>>> dsmLevelProperty = Map.of(property, mappedField); //medicalRecords + Map dsmLevelProperties = Map.of(PROPERTIES, dsmLevelProperty); //properties + Map> dsmLevel = Map.of(DSM_OBJECT, dsmLevelProperties); //dsm + return Map.of(PROPERTIES, dsmLevel); //properties } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Parser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/Parser.java index a7b237a1a..0853ee018 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Parser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/Parser.java @@ -1,7 +1,6 @@ package org.broadinstitute.dsm.model.elastic.export; -import org.broadinstitute.dsm.model.NameValue; public interface Parser { - String parseType(NameValue nameValue); + Object parse(String value); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java index 8d507a688..9b164a537 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java @@ -10,6 +10,12 @@ public class SourceGenerator extends BaseGenerator { + private Parser parser; + + public SourceGenerator(Parser parser) { + this.parser = parser; + } + @Override public Map generate(NameValue nameValue) { initializeNecessaryFields(Objects.requireNonNull(nameValue)); @@ -20,9 +26,13 @@ public Map generate(NameValue nameValue) { protected Map collectExportData() { Map result = new HashMap<>(); String objectKey = TABLE_ALIAS_MAPPINGS.get(dbElement.getTableAlias()); - Map dynamicFieldValues; + Map dynamicFieldValues; try { dynamicFieldValues = new Gson().fromJson((String) nameValue.getValue(), Map.class); + Map transformedMap = new HashMap<>(); + for (Map.Entry entry: dynamicFieldValues.entrySet()) { + transformedMap.put(entry.getKey(), parser.parse((String) entry.getValue())); + } result.put(objectKey, dynamicFieldValues); } catch (JsonSyntaxException jse) { result.put(objectKey, Map.of(dbElement.getColumnName(), nameValue.getValue())); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/TypeParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/TypeParser.java index 1a3d459c9..4f0a8ac4f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/TypeParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/TypeParser.java @@ -1,65 +1,32 @@ package org.broadinstitute.dsm.model.elastic.export; import org.apache.commons.lang3.StringUtils; -import org.broadinstitute.dsm.model.NameValue; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; import java.time.format.DateTimeParseException; -public class TypeParser implements Parser { - public TypeParser() { - } - - @Override - public String parseType(NameValue nameValue) { - String value = (String) nameValue.getValue(); - String type = "text"; +public class TypeParser extends BaseParser { - if (StringUtils.isNumeric(value)) { - type = "integer"; - } else if (isBoolean(value)) { - type = "boolean"; - } else if (isDateOrTimeOrDateTime(value)) { - type = "date"; - } - return type; - } - - boolean isDateOrTimeOrDateTime(String value) { - return isDate(value) || isTime(value) || isDateTime(value); - } - - boolean isDateTime(String value) { - try { - LocalDateTime.parse(value); - } catch (DateTimeParseException dtpe) { - return false; - } - return true; + @Override + protected Object forNumeric(String value) { + return "long"; } - boolean isTime(String value) { - try { - LocalTime.parse(value); - } catch (DateTimeParseException dtpe) { - return false; - } - return true; + @Override + protected Object forBoolean(String value) { + return "boolean"; } - boolean isDate(String value) { - try { - LocalDate.parse(value); - } catch (DateTimeParseException dtpe) { - return false; - } - return true; + @Override + protected Object forDate(String value) { + return "date"; } - boolean isBoolean(String value) { - return value.equalsIgnoreCase("true") || value.equalsIgnoreCase("false"); + @Override + protected Object forString(String value) { + return "text"; } } \ No newline at end of file diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ValueParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ValueParser.java new file mode 100644 index 000000000..038f9cd3b --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ValueParser.java @@ -0,0 +1,24 @@ +package org.broadinstitute.dsm.model.elastic.export; + +public class ValueParser extends BaseParser { + + @Override + protected Object forNumeric(String value) { + return Long.valueOf(value); + } + + @Override + protected Object forBoolean(String value) { + return Boolean.valueOf(value); + } + + @Override + protected Object forDate(String value) { + return value; + } + + @Override + protected Object forString(String value) { + return value; + } +} diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java index aecc05c86..91dd04ef6 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java @@ -11,7 +11,7 @@ public class SourceGeneratorTest { @Test public void generate() { - Generator generator = new TestSourceGenerator(); + Generator generator = new TestSourceGenerator(new ValueParser()); NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "value"); Map objectMap = generator.generate(nameValue); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), SourceGenerator.DSM_OBJECT); @@ -21,6 +21,10 @@ public void generate() { private static class TestSourceGenerator extends SourceGenerator { + public TestSourceGenerator(Parser parser) { + super(parser); + } + @Override protected DBElement getDBElement() { return TestPatchUtil.getColumnNameMap().get(nameValue.getName()); From 747ae27597e70d1ddbbc7bd1e15fdf605998303f Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 21 Oct 2021 11:09:56 +0400 Subject: [PATCH 048/405] DDP-7047 | fix bug in ValueParser.java, ensure that tests run, initialize refactoring --- .../elastic/export/MappingGenerator.java | 22 ++++++++++++------- .../model/elastic/export/SourceGenerator.java | 7 +++--- .../elastic/export/SourceGeneratorTest.java | 10 +++++++++ 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java index 942750514..5a4d2adee 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java @@ -11,6 +11,7 @@ public class MappingGenerator extends BaseGenerator { + public static final String TYPE = "type"; private final Parser parser; public MappingGenerator(Parser parser) { @@ -21,21 +22,26 @@ public MappingGenerator(Parser parser) { public Map generate(NameValue nameValue) { initializeNecessaryFields(Objects.requireNonNull(nameValue)); String property = TABLE_ALIAS_MAPPINGS.get(dbElement.getTableAlias()); + Map> field = collectMapping(nameValue); + Map>> mappedField = Map.of(PROPERTIES, field); + Map>>> dsmLevelProperty = Map.of(property, mappedField); + Map dsmLevelProperties = Map.of(PROPERTIES, dsmLevelProperty); + Map> dsmLevel = Map.of(DSM_OBJECT, dsmLevelProperties); + return Map.of(PROPERTIES, dsmLevel); + } + + private Map> collectMapping(NameValue nameValue) { Map> field = new HashMap<>(); try { Map fieldsByValues = new Gson().fromJson((String) nameValue.getValue(), Map.class); for (Map.Entry entry: fieldsByValues.entrySet()) { Object eachType = parser.parse((String) entry.getValue()); - field.put(entry.getKey(), Map.of("type", eachType)); + field.put(entry.getKey(), Map.of(TYPE, eachType)); } } catch (JsonSyntaxException jse) { - Object type = parser.parse((String)nameValue.getValue()); - field = Map.of(dbElement.getColumnName(), Map.of("type", type)); // ddp_medical_column : {type: "text"} + Object type = parser.parse((String) nameValue.getValue()); + field = Map.of(dbElement.getColumnName(), Map.of(TYPE, type)); } - Map>> mappedField = Map.of(PROPERTIES, field); //properties - Map>>> dsmLevelProperty = Map.of(property, mappedField); //medicalRecords - Map dsmLevelProperties = Map.of(PROPERTIES, dsmLevelProperty); //properties - Map> dsmLevel = Map.of(DSM_OBJECT, dsmLevelProperties); //dsm - return Map.of(PROPERTIES, dsmLevel); //properties + return field; } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java index 9b164a537..6945a9c9c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java @@ -26,16 +26,15 @@ public Map generate(NameValue nameValue) { protected Map collectExportData() { Map result = new HashMap<>(); String objectKey = TABLE_ALIAS_MAPPINGS.get(dbElement.getTableAlias()); - Map dynamicFieldValues; try { - dynamicFieldValues = new Gson().fromJson((String) nameValue.getValue(), Map.class); + Map dynamicFieldValues = new Gson().fromJson((String) nameValue.getValue(), Map.class); Map transformedMap = new HashMap<>(); for (Map.Entry entry: dynamicFieldValues.entrySet()) { transformedMap.put(entry.getKey(), parser.parse((String) entry.getValue())); } - result.put(objectKey, dynamicFieldValues); + result.put(objectKey, transformedMap); } catch (JsonSyntaxException jse) { - result.put(objectKey, Map.of(dbElement.getColumnName(), nameValue.getValue())); + result.put(objectKey, Map.of(dbElement.getColumnName(), parser.parse((String) nameValue.getValue()))); } return result; } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java index 91dd04ef6..8adaf494e 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java @@ -19,6 +19,16 @@ public void generate() { Assert.assertEquals("value", value); } + @Test + public void generateNumeric() { + Generator generator = new TestSourceGenerator(new ValueParser()); + NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "1"); + Map objectMap = generator.generate(nameValue); + Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), SourceGenerator.DSM_OBJECT); + Object value = ((Map)((Map)objectMap.get(SourceGenerator.DSM_OBJECT)).get("medicalRecords")).get(TestPatchUtil.MEDICAL_RECORD_COLUMN); + Assert.assertEquals(1L, value); + } + private static class TestSourceGenerator extends SourceGenerator { public TestSourceGenerator(Parser parser) { From da4dd22b0d2ba14233b652f15b833dd77a52b25b Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 21 Oct 2021 15:09:36 +0400 Subject: [PATCH 049/405] DDP-7047 | eliminate code duplication by introducing template method in parser --- .../model/elastic/export/BaseGenerator.java | 29 +++++++++++++ .../elastic/export/MappingGenerator.java | 42 ++++++++++--------- .../model/elastic/export/SourceGenerator.java | 35 ++++++++-------- .../elastic/export/MappingGeneratorTest.java | 2 +- 4 files changed, 69 insertions(+), 39 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java index 529c3dcb0..f2003cbbe 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java @@ -1,9 +1,12 @@ package org.broadinstitute.dsm.model.elastic.export; +import com.google.gson.Gson; +import com.google.gson.JsonSyntaxException; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.util.PatchUtil; +import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -20,10 +23,16 @@ public abstract class BaseGenerator implements Generator { "p", "participant", "d", "participant" ); + protected static final Gson GSON = new Gson(); + protected final Parser parser; protected NameValue nameValue; protected DBElement dbElement; + public BaseGenerator(Parser parser) { + this.parser = parser; + } + protected void initializeNecessaryFields(NameValue nameValue) { this.nameValue = nameValue; dbElement = getDBElement(); @@ -32,4 +41,24 @@ protected void initializeNecessaryFields(NameValue nameValue) { protected DBElement getDBElement() { return PatchUtil.getColumnNameMap().get(Objects.requireNonNull(nameValue).getName()); } + + protected Map collect() { + Map sourceToUpsert; + try { + sourceToUpsert = parseJson(); + } catch (JsonSyntaxException jse) { + sourceToUpsert = parseSingleElement(); + } + return sourceToUpsert; + } + + protected String getOuterPropertyByAlias() { + return TABLE_ALIAS_MAPPINGS.get(dbElement.getTableAlias()); + } + + protected abstract Map parseJson(); + + protected abstract Map parseSingleElement(); + } + diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java index 5a4d2adee..88a7b332e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java @@ -4,44 +4,46 @@ import java.util.Map; import java.util.Objects; -import com.google.gson.Gson; -import com.google.gson.JsonSyntaxException; import org.broadinstitute.dsm.model.NameValue; public class MappingGenerator extends BaseGenerator { public static final String TYPE = "type"; - private final Parser parser; public MappingGenerator(Parser parser) { - this.parser = parser; + super(parser); } + @Override public Map generate(NameValue nameValue) { initializeNecessaryFields(Objects.requireNonNull(nameValue)); - String property = TABLE_ALIAS_MAPPINGS.get(dbElement.getTableAlias()); - Map> field = collectMapping(nameValue); - Map>> mappedField = Map.of(PROPERTIES, field); - Map>>> dsmLevelProperty = Map.of(property, mappedField); + String property = getOuterPropertyByAlias(); + Map field = collect(); + Map mappedField = Map.of(PROPERTIES, field); + Map dsmLevelProperty = Map.of(property, mappedField); Map dsmLevelProperties = Map.of(PROPERTIES, dsmLevelProperty); Map> dsmLevel = Map.of(DSM_OBJECT, dsmLevelProperties); return Map.of(PROPERTIES, dsmLevel); } - private Map> collectMapping(NameValue nameValue) { - Map> field = new HashMap<>(); - try { - Map fieldsByValues = new Gson().fromJson((String) nameValue.getValue(), Map.class); - for (Map.Entry entry: fieldsByValues.entrySet()) { - Object eachType = parser.parse((String) entry.getValue()); - field.put(entry.getKey(), Map.of(TYPE, eachType)); - } - } catch (JsonSyntaxException jse) { - Object type = parser.parse((String) nameValue.getValue()); - field = Map.of(dbElement.getColumnName(), Map.of(TYPE, type)); + @Override + protected Map parseJson() { + Map resultMap = new HashMap<>(); + Map fieldsByValues = GSON.fromJson((String) nameValue.getValue(), Map.class); + for (Map.Entry entry: fieldsByValues.entrySet()) { + Object eachType = parser.parse((String) entry.getValue()); + resultMap.put(entry.getKey(), Map.of(MappingGenerator.TYPE, eachType)); } - return field; + return resultMap; + } + + @Override + protected Map parseSingleElement() { + Object type = parser.parse((String) nameValue.getValue()); + Map columnWithType = Map.of(dbElement.getColumnName(), Map.of(MappingGenerator.TYPE, type)); + return columnWithType; } + } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java index 6945a9c9c..5ba7f07aa 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java @@ -1,7 +1,5 @@ package org.broadinstitute.dsm.model.elastic.export; -import com.google.gson.Gson; -import com.google.gson.JsonSyntaxException; import org.broadinstitute.dsm.model.NameValue; import java.util.HashMap; @@ -10,33 +8,34 @@ public class SourceGenerator extends BaseGenerator { - private Parser parser; public SourceGenerator(Parser parser) { - this.parser = parser; + super(parser); } + @Override public Map generate(NameValue nameValue) { initializeNecessaryFields(Objects.requireNonNull(nameValue)); - Map mapToExport = collectExportData(); + Map mapToExport = collect(); return Map.of(DSM_OBJECT, mapToExport); } - protected Map collectExportData() { - Map result = new HashMap<>(); - String objectKey = TABLE_ALIAS_MAPPINGS.get(dbElement.getTableAlias()); - try { - Map dynamicFieldValues = new Gson().fromJson((String) nameValue.getValue(), Map.class); - Map transformedMap = new HashMap<>(); - for (Map.Entry entry: dynamicFieldValues.entrySet()) { - transformedMap.put(entry.getKey(), parser.parse((String) entry.getValue())); - } - result.put(objectKey, transformedMap); - } catch (JsonSyntaxException jse) { - result.put(objectKey, Map.of(dbElement.getColumnName(), parser.parse((String) nameValue.getValue()))); + @Override + protected Map parseJson() { + Map dynamicFieldValues = GSON.fromJson((String) nameValue.getValue(), Map.class); + Map transformedMap = new HashMap<>(); + for (Map.Entry entry: dynamicFieldValues.entrySet()) { + transformedMap.put(entry.getKey(), parser.parse((String) entry.getValue())); } - return result; + return Map.of(getOuterPropertyByAlias(), transformedMap); + } + + @Override + protected Map parseSingleElement() { + Map propertyWithFieldValuePair = + Map.of(getOuterPropertyByAlias(), Map.of(dbElement.getColumnName(), parser.parse((String) nameValue.getValue()))); + return propertyWithFieldValuePair; } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java index 98128fb2d..9e5fbf1ac 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java @@ -42,7 +42,7 @@ public void generateIntegerType() { Map objectMap = generator.generate(nameValue); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), BaseGenerator.PROPERTIES); String type = extractDeepestLeveleValue(objectMap); - Assert.assertEquals("integer", type); + Assert.assertEquals("long", type); } @Test From dd62a7e6e86c723418f9653c39cb94d639169156 Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 21 Oct 2021 17:45:15 +0400 Subject: [PATCH 050/405] DDP-7047 | introduce GeneratorPayload in order to distribute appropriate data for creating nested mappings --- .../model/elastic/export/BaseGenerator.java | 41 ++++++++++++++----- .../dsm/model/elastic/export/Generator.java | 2 +- .../elastic/export/GeneratorPayload.java | 22 ++++++++++ .../elastic/export/MappingGenerator.java | 24 ++++++++--- .../model/elastic/export/SourceGenerator.java | 12 ++++-- 5 files changed, 80 insertions(+), 21 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/GeneratorPayload.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java index f2003cbbe..4857eb131 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java @@ -6,7 +6,6 @@ import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.util.PatchUtil; -import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -15,26 +14,27 @@ public abstract class BaseGenerator implements Generator { public static final String DSM_OBJECT = "dsm"; public static final String PROPERTIES = "properties"; - public static final Map TABLE_ALIAS_MAPPINGS = Map.of( - "m", "medicalRecords", - "t", "tissueRecords", - "oD", "oncHistoryDetailRecords", - "r", "participant", - "p", "participant", - "d", "participant" + public static final Map TABLE_ALIAS_MAPPINGS = Map.of( + "m", new PropertyInfo("medicalRecords", true), + "t", new PropertyInfo("tissueRecords", true), + "oD", new PropertyInfo("oncHistoryDetailRecords", true), + "r", new PropertyInfo("participant", false), + "p", new PropertyInfo("participant", false), + "d", new PropertyInfo("participant", false) ); protected static final Gson GSON = new Gson(); protected final Parser parser; protected NameValue nameValue; protected DBElement dbElement; + protected GeneratorPayload generatorPayload; public BaseGenerator(Parser parser) { this.parser = parser; } - protected void initializeNecessaryFields(NameValue nameValue) { - this.nameValue = nameValue; + protected void initializeNecessaryFields() { + this.nameValue = generatorPayload.getNameValue(); dbElement = getDBElement(); } @@ -52,13 +52,32 @@ protected Map collect() { return sourceToUpsert; } - protected String getOuterPropertyByAlias() { + protected PropertyInfo getOuterPropertyByAlias() { return TABLE_ALIAS_MAPPINGS.get(dbElement.getTableAlias()); } protected abstract Map parseJson(); protected abstract Map parseSingleElement(); + + public static class PropertyInfo { + + String propertyName; + boolean isCollection; + + public PropertyInfo(String propertyName, boolean isCollection) { + this.propertyName = Objects.requireNonNull(propertyName); + this.isCollection = isCollection; + } + + public String getPropertyName() { + return propertyName; + } + + public boolean isCollection() { + return isCollection; + } + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Generator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/Generator.java index 75c0907f3..01a478a4a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Generator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/Generator.java @@ -5,5 +5,5 @@ import org.broadinstitute.dsm.model.NameValue; public interface Generator { - Map generate(NameValue nameValue); + Map generate(); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/GeneratorPayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/GeneratorPayload.java new file mode 100644 index 000000000..6400a2662 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/GeneratorPayload.java @@ -0,0 +1,22 @@ +package org.broadinstitute.dsm.model.elastic.export; + +import org.broadinstitute.dsm.model.NameValue; + +public class GeneratorPayload { + + NameValue nameValue; + long recordId; + + public GeneratorPayload(NameValue nameValue, long recordId) { + this.nameValue = nameValue; + this.recordId = recordId; + } + + public NameValue getNameValue() { + return nameValue; + } + + public long getRecordId() { + return recordId; + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java index 88a7b332e..c0c3da3d8 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java @@ -10,6 +10,7 @@ public class MappingGenerator extends BaseGenerator { public static final String TYPE = "type"; + public static final String NESTED = "nested"; public MappingGenerator(Parser parser) { super(parser); @@ -17,17 +18,30 @@ public MappingGenerator(Parser parser) { @Override - public Map generate(NameValue nameValue) { + public Map generate() { initializeNecessaryFields(Objects.requireNonNull(nameValue)); - String property = getOuterPropertyByAlias(); - Map field = collect(); - Map mappedField = Map.of(PROPERTIES, field); - Map dsmLevelProperty = Map.of(property, mappedField); + PropertyInfo propertyInfo = getOuterPropertyByAlias(); + String propertyName = propertyInfo.getPropertyName(); + Map mappedField = buildMappedField(); + Map dsmLevelProperty = Map.of(propertyName, mappedField); Map dsmLevelProperties = Map.of(PROPERTIES, dsmLevelProperty); Map> dsmLevel = Map.of(DSM_OBJECT, dsmLevelProperties); return Map.of(PROPERTIES, dsmLevel); } + private Map buildMappedField() { + PropertyInfo propertyInfo = getOuterPropertyByAlias(); + boolean isPropertyCollection = propertyInfo.isCollection(); + Map field = collect(); + Map mappedField; + if (isPropertyCollection) { + mappedField = Map.of(TYPE, NESTED, PROPERTIES, field); + } else { + mappedField = Map.of(PROPERTIES, field); + } + return mappedField; + } + @Override protected Map parseJson() { Map resultMap = new HashMap<>(); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java index 5ba7f07aa..a83653aa2 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java @@ -9,13 +9,14 @@ public class SourceGenerator extends BaseGenerator { - public SourceGenerator(Parser parser) { + public SourceGenerator(Parser parser, GeneratorPayload generatorPayload) { super(parser); + } @Override - public Map generate(NameValue nameValue) { + public Map generate() { initializeNecessaryFields(Objects.requireNonNull(nameValue)); Map mapToExport = collect(); return Map.of(DSM_OBJECT, mapToExport); @@ -25,14 +26,17 @@ public Map generate(NameValue nameValue) { protected Map parseJson() { Map dynamicFieldValues = GSON.fromJson((String) nameValue.getValue(), Map.class); Map transformedMap = new HashMap<>(); - for (Map.Entry entry: dynamicFieldValues.entrySet()) { + for (Map.Entry entry : dynamicFieldValues.entrySet()) { transformedMap.put(entry.getKey(), parser.parse((String) entry.getValue())); } - return Map.of(getOuterPropertyByAlias(), transformedMap); + PropertyInfo outerPropertyByAlias = getOuterPropertyByAlias(); + Map.of(getOuterPropertyByAlias(), transformedMap); + return null; } @Override protected Map parseSingleElement() { + PropertyInfo outerPropertyByAlias = getOuterPropertyByAlias(); Map propertyWithFieldValuePair = Map.of(getOuterPropertyByAlias(), Map.of(dbElement.getColumnName(), parser.parse((String) nameValue.getValue()))); return propertyWithFieldValuePair; From 38ed639bfee0aa15b0bcd0f5452d7341f753e13a Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 22 Oct 2021 10:40:30 +0400 Subject: [PATCH 051/405] DDP-7047 | refactor code, add generator payload, add tests for mapper and sourcer, add util class for elastic search, rename elastic searchable interface method for searching doc by id --- .../dsm/model/elastic/Util.java | 20 ++++++ .../model/elastic/export/BaseGenerator.java | 19 +++--- .../model/elastic/export/ExportFacade.java | 18 +++-- .../elastic/export/ExportFacadePayload.java | 18 ++--- .../elastic/export/MappingGenerator.java | 11 ++- .../model/elastic/export/SourceGenerator.java | 18 ++--- .../model/elastic/search/ElasticSearch.java | 10 +-- .../elastic/search/ElasticSearchable.java | 2 +- .../dsm/model/patch/BasePatch.java | 3 +- .../dsm/route/ClinicalKitsRoute.java | 2 +- .../dsm/route/KitUploadRoute.java | 8 +-- .../dsm/util/ParticipantUtil.java | 4 ++ .../elastic/export/MappingGeneratorTest.java | 68 +++++++++++++------ .../elastic/export/SourceGeneratorTest.java | 31 +++++++-- .../model/elastic/export/TestPatchUtil.java | 4 +- .../participant/ParticipantWrapperTest.java | 2 +- 16 files changed, 149 insertions(+), 89 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/Util.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java new file mode 100644 index 000000000..7fa4bfb46 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -0,0 +1,20 @@ +package org.broadinstitute.dsm.model.elastic; + +import org.broadinstitute.dsm.util.ParticipantUtil; + +public class Util { + + public static String getQueryTypeFromId(String id) { + String type; + if (ParticipantUtil.isHruid(id)) { + type = "profile.hruid"; + } else if (ParticipantUtil.isGuid(id)){ + type = "profile.guid"; + } else if (ParticipantUtil.isLegacyAltPid(id)) { + type = "profile.legacyAltPid"; + } else { + type = "profile.legacyShortId"; + } + return type; + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java index 4857eb131..d268a25cb 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java @@ -24,22 +24,19 @@ public abstract class BaseGenerator implements Generator { ); protected static final Gson GSON = new Gson(); protected final Parser parser; - - protected NameValue nameValue; - protected DBElement dbElement; protected GeneratorPayload generatorPayload; - public BaseGenerator(Parser parser) { - this.parser = parser; + public BaseGenerator(Parser parser, GeneratorPayload generatorPayload) { + this.parser = Objects.requireNonNull(parser); + this.generatorPayload = Objects.requireNonNull(generatorPayload); } - protected void initializeNecessaryFields() { - this.nameValue = generatorPayload.getNameValue(); - dbElement = getDBElement(); + protected DBElement getDBElement() { + return PatchUtil.getColumnNameMap().get(Objects.requireNonNull(getNameValue()).getName()); } - protected DBElement getDBElement() { - return PatchUtil.getColumnNameMap().get(Objects.requireNonNull(nameValue).getName()); + protected NameValue getNameValue() { + return generatorPayload.getNameValue(); } protected Map collect() { @@ -53,7 +50,7 @@ protected Map collect() { } protected PropertyInfo getOuterPropertyByAlias() { - return TABLE_ALIAS_MAPPINGS.get(dbElement.getTableAlias()); + return TABLE_ALIAS_MAPPINGS.get(getDBElement().getTableAlias()); } protected abstract Map parseJson(); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index fc7622595..1a46f5624 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -1,37 +1,41 @@ package org.broadinstitute.dsm.model.elastic.export; import java.util.Map; +import java.util.Objects; + +import org.broadinstitute.dsm.model.elastic.search.ElasticSearchable; public class ExportFacade { BaseExporter exportable; Generator generator; + ElasticSearchable searchable; private ExportFacadePayload exportFacadePayload; public ExportFacade(ExportFacadePayload exportFacadePayload) { - this.exportFacadePayload = exportFacadePayload; + this.exportFacadePayload = Objects.requireNonNull(exportFacadePayload); exportable = new ElasticExportAdapter(); - generator = new MappingGenerator(new TypeParser()); } public void export() { upsertMapping(); + fetchData(); upsertData(); } private void upsertMapping() { - generator = new MappingGenerator(new TypeParser()); - Map mappingToUpsert = generator.generate(exportFacadePayload.getNameValue()); + generator = new MappingGenerator(new TypeParser(), exportFacadePayload.getGeneratorPayload()); + Map mappingToUpsert = generator.generate(); UpsertMappingRequestPayload upsertMappingRequestPayload = new UpsertMappingRequestPayload(exportFacadePayload.getIndex()); exportable.setUpsertMappingRequestPayload(upsertMappingRequestPayload); exportable.exportMapping(mappingToUpsert); } private void upsertData() { - generator = new SourceGenerator(new ValueParser()); - Map elasticDataToExport = generator.generate(exportFacadePayload.getNameValue()); + generator = new SourceGenerator(new ValueParser(), exportFacadePayload.getGeneratorPayload()); + Map elasticDataToExport = generator.generate(); UpsertDataRequestPayload upsertDataRequestPayload = new UpsertDataRequestPayload.Builder(exportFacadePayload.getIndex(), - exportFacadePayload.getId()) + exportFacadePayload.getDocId()) .withDocAsUpsert(true) .withRetryOnConflict(5) .build(); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java index c404a7749..129cbb225 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java @@ -6,24 +6,24 @@ public class ExportFacadePayload { private String index; - private String id; - private NameValue nameValue; + private String docId; + private GeneratorPayload generatorPayload; - public ExportFacadePayload(String index, String id, NameValue nameValue) { + public ExportFacadePayload(String index, String docId, GeneratorPayload generatorPayload) { this.index = Objects.requireNonNull(index); - this.id = Objects.requireNonNull(id); - this.nameValue = Objects.requireNonNull(nameValue); + this.docId = Objects.requireNonNull(docId); + this.generatorPayload = Objects.requireNonNull(generatorPayload); } public String getIndex() { return index; } - public String getId() { - return id; + public String getDocId() { + return docId; } - public NameValue getNameValue() { - return nameValue; + public GeneratorPayload getGeneratorPayload() { + return generatorPayload; } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java index c0c3da3d8..28bb89360 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java @@ -12,14 +12,13 @@ public class MappingGenerator extends BaseGenerator { public static final String TYPE = "type"; public static final String NESTED = "nested"; - public MappingGenerator(Parser parser) { - super(parser); + public MappingGenerator(Parser parser, GeneratorPayload generatorPayload) { + super(parser, generatorPayload); } @Override public Map generate() { - initializeNecessaryFields(Objects.requireNonNull(nameValue)); PropertyInfo propertyInfo = getOuterPropertyByAlias(); String propertyName = propertyInfo.getPropertyName(); Map mappedField = buildMappedField(); @@ -45,7 +44,7 @@ private Map buildMappedField() { @Override protected Map parseJson() { Map resultMap = new HashMap<>(); - Map fieldsByValues = GSON.fromJson((String) nameValue.getValue(), Map.class); + Map fieldsByValues = GSON.fromJson((String) getNameValue().getValue(), Map.class); for (Map.Entry entry: fieldsByValues.entrySet()) { Object eachType = parser.parse((String) entry.getValue()); resultMap.put(entry.getKey(), Map.of(MappingGenerator.TYPE, eachType)); @@ -55,8 +54,8 @@ protected Map parseJson() { @Override protected Map parseSingleElement() { - Object type = parser.parse((String) nameValue.getValue()); - Map columnWithType = Map.of(dbElement.getColumnName(), Map.of(MappingGenerator.TYPE, type)); + Object type = parser.parse((String) getNameValue().getValue()); + Map columnWithType = Map.of(getDBElement().getColumnName(), Map.of(MappingGenerator.TYPE, type)); return columnWithType; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java index a83653aa2..17679b8b1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java @@ -10,35 +10,31 @@ public class SourceGenerator extends BaseGenerator { public SourceGenerator(Parser parser, GeneratorPayload generatorPayload) { - super(parser); - + super(parser, generatorPayload); } @Override public Map generate() { - initializeNecessaryFields(Objects.requireNonNull(nameValue)); - Map mapToExport = collect(); - return Map.of(DSM_OBJECT, mapToExport); + Map dataToExport = collect(); + return Map.of(DSM_OBJECT, dataToExport); } @Override protected Map parseJson() { - Map dynamicFieldValues = GSON.fromJson((String) nameValue.getValue(), Map.class); + Map dynamicFieldValues = GSON.fromJson((String) getNameValue().getValue(), Map.class); Map transformedMap = new HashMap<>(); for (Map.Entry entry : dynamicFieldValues.entrySet()) { transformedMap.put(entry.getKey(), parser.parse((String) entry.getValue())); } - PropertyInfo outerPropertyByAlias = getOuterPropertyByAlias(); - Map.of(getOuterPropertyByAlias(), transformedMap); - return null; + return Map.of(getOuterPropertyByAlias().getPropertyName(), transformedMap); } @Override protected Map parseSingleElement() { - PropertyInfo outerPropertyByAlias = getOuterPropertyByAlias(); Map propertyWithFieldValuePair = - Map.of(getOuterPropertyByAlias(), Map.of(dbElement.getColumnName(), parser.parse((String) nameValue.getValue()))); + Map.of(getOuterPropertyByAlias().getPropertyName(), Map.of(getDBElement().getColumnName(), + parser.parse((String) getNameValue().getValue()))); return propertyWithFieldValuePair; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java index 14f960dde..37abcee0c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java @@ -13,6 +13,7 @@ import com.google.gson.Gson; import lombok.Setter; import org.apache.commons.lang3.StringUtils; +import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.broadinstitute.dsm.util.ParticipantUtil; import org.elasticsearch.action.search.SearchRequest; @@ -193,14 +194,9 @@ public ElasticSearch getParticipantsByRangeAndIds(String participantIndexES, int } @Override - public ElasticSearchParticipantDto getParticipantByShortId(String esParticipantsIndex, String shortId) { - String type = ""; + public ElasticSearchParticipantDto getParticipantById(String esParticipantsIndex, String shortId) { + String type = Util.getQueryTypeFromId(shortId); String id = Objects.requireNonNull(shortId); - if (ParticipantUtil.isHruid(shortId)) { - type = "profile.hruid"; - } else { - type = "profile.legacyShortId"; - } SearchRequest searchRequest = new SearchRequest(Objects.requireNonNull(esParticipantsIndex)); TermQueryBuilder shortIdQuery = QueryBuilders.termQuery(type, id); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchable.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchable.java index 4aef55310..ff7c09e2f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchable.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchable.java @@ -15,7 +15,7 @@ public interface ElasticSearchable { ElasticSearch getParticipantsByRangeAndIds(String participantIndexES, int from, int to, List participantIds); - ElasticSearchParticipantDto getParticipantByShortId(String esParticipantsIndex, String shortId); + ElasticSearchParticipantDto getParticipantById(String esParticipantsIndex, String id); ElasticSearch getAllParticipantsDataByInstanceIndex(String esParticipantsIndex); diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index 828e2664a..ee8bfc10b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -69,8 +69,9 @@ protected BasePatch(Patch patch) { } protected void exportToES(NameValue nameValue) { + GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, Integer.parseInt(patch.getId())); ExportFacadePayload exportFacadePayload = - new ExportFacadePayload(ddpInstance.getParticipantIndexES(), patch.getDdpParticipantId(), nameValue); + new ExportFacadePayload(ddpInstance.getParticipantIndexES(), patch.getDdpParticipantId(), generatorPayload); ExportFacade exportFacade = new ExportFacade(exportFacadePayload); exportFacade.export(); } diff --git a/src/main/java/org/broadinstitute/dsm/route/ClinicalKitsRoute.java b/src/main/java/org/broadinstitute/dsm/route/ClinicalKitsRoute.java index e73456495..cb8d31ceb 100644 --- a/src/main/java/org/broadinstitute/dsm/route/ClinicalKitsRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/ClinicalKitsRoute.java @@ -78,7 +78,7 @@ private ClinicalKitDto getClinicalKit(String kitLabel) { hruid = maybeBspKitQueryResult.getBspParticipantId().substring(maybeBspKitQueryResult.getBspParticipantId().lastIndexOf('_') + 1); } ElasticSearchParticipantDto participantByShortId = - new ElasticSearch().getParticipantByShortId(ddpInstance.getParticipantIndexES(), hruid); + new ElasticSearch().getParticipantById(ddpInstance.getParticipantIndexES(), hruid); setNeccessaryDataToClinicalKit(clinicalKit, participantByShortId); }); maybeKitInfo.orElseThrow(); diff --git a/src/main/java/org/broadinstitute/dsm/route/KitUploadRoute.java b/src/main/java/org/broadinstitute/dsm/route/KitUploadRoute.java index 5a25c7826..b373e879f 100644 --- a/src/main/java/org/broadinstitute/dsm/route/KitUploadRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/KitUploadRoute.java @@ -216,7 +216,7 @@ private void uploadKit(@NonNull DDPInstance ddpInstance, @NonNull KitType kitTyp //if kit has ddpParticipantId use that (RGP!) if (StringUtils.isBlank(kit.getParticipantId())) { ElasticSearchParticipantDto participantByShortId = - elasticSearch.getParticipantByShortId(ddpInstance.getParticipantIndexES(), kit.getShortId()); + elasticSearch.getParticipantById(ddpInstance.getParticipantIndexES(), kit.getShortId()); participantGuid = participantByShortId.getProfile().map(ESProfile::getParticipantGuid).orElse(""); participantLegacyAltPid = participantByShortId.getProfile().map(ESProfile::getParticipantLegacyAltPid).orElse(""); kit.setParticipantId(!participantGuid.isEmpty() ? participantGuid : participantLegacyAltPid); @@ -334,8 +334,8 @@ private void handleKit(@NonNull Connection conn, @NonNull DDPInstance ddpInstanc } } else { - String participantGuid = elasticSearch.getParticipantByShortId(ddpInstance.getParticipantIndexES(), kit.getShortId()).getProfile().map(ESProfile::getParticipantGuid).orElse(""); - String participantLegacyAltPid = elasticSearch.getParticipantByShortId(ddpInstance.getParticipantIndexES(), kit.getShortId()).getProfile().map(ESProfile::getParticipantLegacyAltPid).orElse(""); + String participantGuid = elasticSearch.getParticipantById(ddpInstance.getParticipantIndexES(), kit.getShortId()).getProfile().map(ESProfile::getParticipantGuid).orElse(""); + String participantLegacyAltPid = elasticSearch.getParticipantById(ddpInstance.getParticipantIndexES(), kit.getShortId()).getProfile().map(ESProfile::getParticipantLegacyAltPid).orElse(""); if (checkAndSetParticipantIdIfKitExists(ddpInstance, conn, kit, participantGuid, participantLegacyAltPid, kitType.getKitTypeId()) && !uploadAnyway) { duplicateKitList.add(kit); } @@ -504,7 +504,7 @@ private String userExistsInRealm(DDPInstance ddpInstanceByRealm, ElasticSearchParticipantDto participantByShortId; try { - participantByShortId = elasticSearch.getParticipantByShortId(ddpInstanceByRealm.getParticipantIndexES(), participantIdFromDoc); + participantByShortId = elasticSearch.getParticipantById(ddpInstanceByRealm.getParticipantIndexES(), participantIdFromDoc); } catch (Exception e) { throw new RuntimeException("Participant " + participantIdFromDoc + " does not belong to this study", e); } diff --git a/src/main/java/org/broadinstitute/dsm/util/ParticipantUtil.java b/src/main/java/org/broadinstitute/dsm/util/ParticipantUtil.java index 9a25cb879..d0392a01a 100644 --- a/src/main/java/org/broadinstitute/dsm/util/ParticipantUtil.java +++ b/src/main/java/org/broadinstitute/dsm/util/ParticipantUtil.java @@ -27,6 +27,10 @@ public static boolean isGuid(@NonNull String participantId) { return participantId.length() == 20; } + public static boolean isLegacyAltPid(@NonNull String participantId) { + return participantId.length() > 20; + } + public static boolean matchesApplicantEmail(ESProfile applicantProfile, Map applicantDataMap, Map participantDataMap) { diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java index 9e5fbf1ac..4a4b7afe4 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java @@ -11,17 +11,14 @@ public class MappingGeneratorTest { - static Generator generator; - - @BeforeClass - public static void setUp() { - generator = new TestMappingGenerator(new TypeParser()); - } @Test public void generateTextType() { - NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "value"); - Map objectMap = generator.generate(nameValue); + GeneratorPayload generatorPayload = new GeneratorPayload( + new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "value"), + 0L + ); + Map objectMap = TestMappingGenerator.of(generatorPayload).generate(); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), BaseGenerator.PROPERTIES); String type = extractDeepestLeveleValue(objectMap); Assert.assertEquals("text", type); @@ -29,8 +26,11 @@ public void generateTextType() { @Test public void generateBooleanType() { - NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "true"); - Map objectMap = generator.generate(nameValue); + GeneratorPayload generatorPayload = new GeneratorPayload( + new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "true"), + 0L + ); + Map objectMap = TestMappingGenerator.of(generatorPayload).generate(); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), BaseGenerator.PROPERTIES); String type = extractDeepestLeveleValue(objectMap); Assert.assertEquals("boolean", type); @@ -38,8 +38,11 @@ public void generateBooleanType() { @Test public void generateIntegerType() { - NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "45"); - Map objectMap = generator.generate(nameValue); + GeneratorPayload generatorPayload = new GeneratorPayload( + new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "45"), + 0L + ); + Map objectMap = TestMappingGenerator.of(generatorPayload).generate(); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), BaseGenerator.PROPERTIES); String type = extractDeepestLeveleValue(objectMap); Assert.assertEquals("long", type); @@ -47,39 +50,60 @@ public void generateIntegerType() { @Test public void generateDateType() { - NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "2021-10-30"); - Map objectMap = generator.generate(nameValue); + GeneratorPayload generatorPayload = new GeneratorPayload( + new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "2021-10-30"), + 0L + ); + Map objectMap = TestMappingGenerator.of(generatorPayload).generate(); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), BaseGenerator.PROPERTIES); String type = extractDeepestLeveleValue(objectMap); Assert.assertEquals("date", type); } + @Test + public void generateNestedType() { + GeneratorPayload generatorPayload = new GeneratorPayload( + new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "2021-10-30"), + 100L + ); + Map objectMap = TestMappingGenerator.of(generatorPayload).generate(); + Assert.assertEquals(MappingGenerator.NESTED, getMedicalRecordProperty(objectMap).get(MappingGenerator.TYPE)); + } + private String extractDeepestLeveleValue(Map objectMap) { return (String) ((Map) ((Map) - ((Map) + getMedicalRecordProperty(objectMap) + .get(BaseGenerator.PROPERTIES)) + .get("medical_record_column")) + .get("type"); + } + + private Map getMedicalRecordProperty(Map objectMap) { + return (Map) ((Map) ((Map) ((Map) objectMap .get(BaseGenerator.PROPERTIES)) .get(BaseGenerator.DSM_OBJECT)) .get(BaseGenerator.PROPERTIES)) - .get("medicalRecords")) - .get(BaseGenerator.PROPERTIES)) - .get("medical_record_column")) - .get("type"); + .get("medicalRecords"); } private static class TestMappingGenerator extends MappingGenerator { - public TestMappingGenerator(Parser typeParser) { - super(typeParser); + public TestMappingGenerator(Parser typeParser, GeneratorPayload generatorPayload) { + super(typeParser, generatorPayload); } @Override protected DBElement getDBElement() { - return TestPatchUtil.getColumnNameMap().get(nameValue.getName()); + return TestPatchUtil.getColumnNameMap().get(getNameValue().getName()); + } + + public static TestMappingGenerator of(GeneratorPayload generatorPayload) { + return new TestMappingGenerator(new TypeParser(), generatorPayload); } } } \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java index 8adaf494e..c2f532e7e 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java @@ -11,9 +11,10 @@ public class SourceGeneratorTest { @Test public void generate() { - Generator generator = new TestSourceGenerator(new ValueParser()); NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "value"); - Map objectMap = generator.generate(nameValue); + GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, 0L); + Generator generator = new TestSourceGenerator(new ValueParser(), generatorPayload); + Map objectMap = generator.generate(); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), SourceGenerator.DSM_OBJECT); String value = (String)((Map)((Map)objectMap.get(SourceGenerator.DSM_OBJECT)).get("medicalRecords")).get(TestPatchUtil.MEDICAL_RECORD_COLUMN); Assert.assertEquals("value", value); @@ -21,23 +22,39 @@ public void generate() { @Test public void generateNumeric() { - Generator generator = new TestSourceGenerator(new ValueParser()); NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "1"); - Map objectMap = generator.generate(nameValue); + GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, 0L); + Generator generator = new TestSourceGenerator(new ValueParser(), generatorPayload); + Map objectMap = generator.generate(); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), SourceGenerator.DSM_OBJECT); Object value = ((Map)((Map)objectMap.get(SourceGenerator.DSM_OBJECT)).get("medicalRecords")).get(TestPatchUtil.MEDICAL_RECORD_COLUMN); Assert.assertEquals(1L, value); } + @Test + public void generateFromJson() { + NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "{\"DDP_INSTANCE\": \"TEST\"}"); + GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, 0L); + Generator generator = new TestSourceGenerator(new ValueParser(), generatorPayload); + Map objectMap = generator.generate(); + Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), SourceGenerator.DSM_OBJECT); + Object value = + ((Map)((Map)objectMap + .get(SourceGenerator.DSM_OBJECT)) + .get("medicalRecords")) + .get("DDP_INSTANCE"); + Assert.assertEquals("TEST", value); + } + private static class TestSourceGenerator extends SourceGenerator { - public TestSourceGenerator(Parser parser) { - super(parser); + public TestSourceGenerator(Parser parser, GeneratorPayload generatorPayload) { + super(parser,generatorPayload); } @Override protected DBElement getDBElement() { - return TestPatchUtil.getColumnNameMap().get(nameValue.getName()); + return TestPatchUtil.getColumnNameMap().get(getNameValue().getName()); } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/TestPatchUtil.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/TestPatchUtil.java index 66b18d772..dc9172440 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/TestPatchUtil.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/TestPatchUtil.java @@ -11,7 +11,9 @@ class TestPatchUtil extends PatchUtil { public static Map getColumnNameMap() { DBElement dbElement = new DBElement("ddp_medical_record", "m", "pr", MEDICAL_RECORD_COLUMN); - return Map.of(MEDICAL_RECORD_COLUMN, dbElement); + return Map.of( + MEDICAL_RECORD_COLUMN, dbElement + ); } } diff --git a/src/test/java/org/broadinstitute/dsm/model/participant/ParticipantWrapperTest.java b/src/test/java/org/broadinstitute/dsm/model/participant/ParticipantWrapperTest.java index 890734343..5d117fb19 100644 --- a/src/test/java/org/broadinstitute/dsm/model/participant/ParticipantWrapperTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/participant/ParticipantWrapperTest.java @@ -131,7 +131,7 @@ public ElasticSearch getParticipantsByRangeAndIds(String participantIndexES, int } @Override - public ElasticSearchParticipantDto getParticipantByShortId(String esParticipantsIndex, String shortId) { + public ElasticSearchParticipantDto getParticipantById(String esParticipantsIndex, String id) { return null; } From 8f54ab4babf38790fd0d485922364b5d137eede2 Mon Sep 17 00:00:00 2001 From: Nika Date: Fri, 22 Oct 2021 11:04:47 +0400 Subject: [PATCH 052/405] DDP-7047 | start implementing fetchData() method in ExportFacade --- .../org/broadinstitute/dsm/model/elastic/ESDsm.java | 3 +++ .../dsm/model/elastic/export/ExportFacade.java | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java index 8967c3710..c9465c80a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java @@ -37,4 +37,7 @@ public class ESDsm { @SerializedName(ESObjectConstants.TISSUE_RECORDS) List> tissueRecords; + @SerializedName(ESObjectConstants.MEDICAL_RECORDS) + List> medicalRecords; + } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 1a46f5624..99f5f3e33 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -3,6 +3,9 @@ import java.util.Map; import java.util.Objects; +import org.broadinstitute.dsm.model.elastic.ESDsm; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchable; public class ExportFacade { @@ -15,6 +18,7 @@ public class ExportFacade { public ExportFacade(ExportFacadePayload exportFacadePayload) { this.exportFacadePayload = Objects.requireNonNull(exportFacadePayload); exportable = new ElasticExportAdapter(); + searchable = new ElasticSearch(); } public void export() { @@ -31,6 +35,14 @@ private void upsertMapping() { exportable.exportMapping(mappingToUpsert); } + private void fetchData() { + ElasticSearchParticipantDto participantById = searchable.getParticipantById(exportFacadePayload.getIndex(), exportFacadePayload.getDocId()); + participantById.getDsm().map(ESDsm::getMedicalRecords) + .ifPresent(medicalRecords -> { + medicalRecords.stream().map(eachRecord -> eachRecor) + }); + } + private void upsertData() { generator = new SourceGenerator(new ValueParser(), exportFacadePayload.getGeneratorPayload()); Map elasticDataToExport = generator.generate(); From 9c61800f31bd0880cfb944a41a8b1993e774782d Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 22 Oct 2021 15:32:41 +0400 Subject: [PATCH 053/405] DDP-7047 | add id for collection objects in mapping generator and source generator --- .../dsm/model/elastic/export/BaseGenerator.java | 1 + .../dsm/model/elastic/export/ExportFacade.java | 8 ++++---- .../model/elastic/export/MappingGenerator.java | 16 +++++++++++----- .../model/elastic/export/SourceGenerator.java | 12 +++++++++--- .../elastic/export/MappingGeneratorTest.java | 14 +++++++++++++- 5 files changed, 38 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java index d268a25cb..38e746f63 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java @@ -22,6 +22,7 @@ public abstract class BaseGenerator implements Generator { "p", new PropertyInfo("participant", false), "d", new PropertyInfo("participant", false) ); + public static final String ID = "id"; protected static final Gson GSON = new Gson(); protected final Parser parser; protected GeneratorPayload generatorPayload; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 99f5f3e33..3eda770ae 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -37,10 +37,10 @@ private void upsertMapping() { private void fetchData() { ElasticSearchParticipantDto participantById = searchable.getParticipantById(exportFacadePayload.getIndex(), exportFacadePayload.getDocId()); - participantById.getDsm().map(ESDsm::getMedicalRecords) - .ifPresent(medicalRecords -> { - medicalRecords.stream().map(eachRecord -> eachRecor) - }); +// participantById.getDsm().map(ESDsm::getMedicalRecords) +// .ifPresent(medicalRecords -> { +// medicalRecords.stream().map(eachRecord -> eachRecor) +// }); } private void upsertData() { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java index 28bb89360..7be614f6d 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java @@ -2,15 +2,13 @@ import java.util.HashMap; import java.util.Map; -import java.util.Objects; - -import org.broadinstitute.dsm.model.NameValue; public class MappingGenerator extends BaseGenerator { public static final String TYPE = "type"; public static final String NESTED = "nested"; + public static final String TYPE_KEYWORD = "keyword"; public MappingGenerator(Parser parser, GeneratorPayload generatorPayload) { super(parser, generatorPayload); @@ -55,8 +53,16 @@ protected Map parseJson() { @Override protected Map parseSingleElement() { Object type = parser.parse((String) getNameValue().getValue()); - Map columnWithType = Map.of(getDBElement().getColumnName(), Map.of(MappingGenerator.TYPE, type)); - return columnWithType; + return getFieldWithType(type); + } + + protected Map getFieldWithType(Object type) { + return getOuterPropertyByAlias().isCollection() + ? Map.of( + ID, Map.of(TYPE, TYPE_KEYWORD), + getDBElement().getColumnName(), Map.of(MappingGenerator.TYPE, type) + ) + : Map.of(getDBElement().getColumnName(), Map.of(MappingGenerator.TYPE, type)); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java index 17679b8b1..df496348e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java @@ -32,9 +32,15 @@ protected Map parseJson() { @Override protected Map parseSingleElement() { - Map propertyWithFieldValuePair = - Map.of(getOuterPropertyByAlias().getPropertyName(), Map.of(getDBElement().getColumnName(), - parser.parse((String) getNameValue().getValue()))); + boolean isCollection = getOuterPropertyByAlias().isCollection(); + Map propertyWithFieldValuePair = null; + if (isCollection) { + propertyWithFieldValuePair = Map.of( + getOuterPropertyByAlias().getPropertyName(), Map.of(getDBElement().getColumnName(), + parser.parse((String) getNameValue().getValue())), + ID, generatorPayload.getRecordId() + ); + } return propertyWithFieldValuePair; } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java index 4a4b7afe4..f5cd52054 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java @@ -4,7 +4,6 @@ import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; import org.junit.Assert; -import org.junit.BeforeClass; import org.junit.Test; import java.util.Map; @@ -70,6 +69,19 @@ public void generateNestedType() { Assert.assertEquals(MappingGenerator.NESTED, getMedicalRecordProperty(objectMap).get(MappingGenerator.TYPE)); } + @Test + public void getFieldWithTypeCollectionTrue() { + GeneratorPayload generatorPayload = new GeneratorPayload( + new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "2021-10-30"), + 100L + ); + TestMappingGenerator generator = TestMappingGenerator.of(generatorPayload); + generator.getOuterPropertyByAlias().isCollection = true; + Object type = generator.parser.parse((String) generator.getNameValue().getValue()); + Map fieldWithType = generator.getFieldWithType(type); + Assert.assertTrue(fieldWithType.containsKey(BaseGenerator.ID)); + } + private String extractDeepestLeveleValue(Map objectMap) { return (String) ((Map) From 2184bac7894c7aaece615d5565cd0c03d514c2c8 Mon Sep 17 00:00:00 2001 From: Nika Date: Fri, 22 Oct 2021 16:45:28 +0400 Subject: [PATCH 054/405] DDP-7047 | add template method in BaseGenerator in order to remove code duplication, refactor child Generator classes accordingly --- .../model/elastic/export/BaseGenerator.java | 25 +++++++++++++---- .../model/elastic/export/ExportFacade.java | 8 +++--- .../elastic/export/MappingGenerator.java | 16 +++++++++-- .../model/elastic/export/SourceGenerator.java | 27 ++++++++++++------- .../elastic/export/MappingGeneratorTest.java | 3 +-- 5 files changed, 56 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java index 38e746f63..1c35d7dad 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java @@ -39,7 +39,11 @@ protected DBElement getDBElement() { protected NameValue getNameValue() { return generatorPayload.getNameValue(); } - + + protected PropertyInfo getOuterPropertyByAlias() { + return TABLE_ALIAS_MAPPINGS.get(getDBElement().getTableAlias()); + } + protected Map collect() { Map sourceToUpsert; try { @@ -50,14 +54,25 @@ protected Map collect() { return sourceToUpsert; } - protected PropertyInfo getOuterPropertyByAlias() { - return TABLE_ALIAS_MAPPINGS.get(getDBElement().getTableAlias()); - } - protected abstract Map parseJson(); protected abstract Map parseSingleElement(); + protected Map getFieldWithElement() { + Map fieldElementMap; + Object element = parser.parse((String) getNameValue().getValue()); + if (getOuterPropertyByAlias().isCollection) { + fieldElementMap = getElementWithId(element); + } else { + fieldElementMap = getElement(element); + } + return fieldElementMap; + } + + protected abstract Map getElementWithId(Object element); + + protected abstract Map getElement(Object element); + public static class PropertyInfo { String propertyName; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 3eda770ae..19248c919 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -37,10 +37,10 @@ private void upsertMapping() { private void fetchData() { ElasticSearchParticipantDto participantById = searchable.getParticipantById(exportFacadePayload.getIndex(), exportFacadePayload.getDocId()); -// participantById.getDsm().map(ESDsm::getMedicalRecords) -// .ifPresent(medicalRecords -> { -// medicalRecords.stream().map(eachRecord -> eachRecor) -// }); + participantById.getDsm().map(ESDsm::getMedicalRecords) + .ifPresent(medicalRecords -> { + medicalRecords.stream().map() + }); } private void upsertData() { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java index 7be614f6d..028cad1e3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java @@ -52,8 +52,20 @@ protected Map parseJson() { @Override protected Map parseSingleElement() { - Object type = parser.parse((String) getNameValue().getValue()); - return getFieldWithType(type); + return getFieldWithElement(); + } + + @Override + protected Map getElementWithId(Object type) { + return Map.of( + ID, Map.of(TYPE, TYPE_KEYWORD), + getDBElement().getColumnName(), Map.of(MappingGenerator.TYPE, type) + ); + } + + @Override + protected Map getElement(Object type) { + return Map.of(getDBElement().getColumnName(), Map.of(MappingGenerator.TYPE, type)); } protected Map getFieldWithType(Object type) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java index df496348e..aa6633e72 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java @@ -32,16 +32,23 @@ protected Map parseJson() { @Override protected Map parseSingleElement() { - boolean isCollection = getOuterPropertyByAlias().isCollection(); - Map propertyWithFieldValuePair = null; - if (isCollection) { - propertyWithFieldValuePair = Map.of( - getOuterPropertyByAlias().getPropertyName(), Map.of(getDBElement().getColumnName(), - parser.parse((String) getNameValue().getValue())), - ID, generatorPayload.getRecordId() - ); - } - return propertyWithFieldValuePair; + return getFieldWithElement(); + } + + @Override + protected Map getElementWithId(Object element) { + return Map.of( + getOuterPropertyByAlias().getPropertyName(), Map.of(getDBElement().getColumnName(), + element), + ID, generatorPayload.getRecordId()); } + @Override + protected Map getElement(Object element) { + return Map.of( + getOuterPropertyByAlias().getPropertyName(), Map.of(getDBElement().getColumnName(), + element)); + } + + } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java index f5cd52054..879feaa7f 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java @@ -77,8 +77,7 @@ public void getFieldWithTypeCollectionTrue() { ); TestMappingGenerator generator = TestMappingGenerator.of(generatorPayload); generator.getOuterPropertyByAlias().isCollection = true; - Object type = generator.parser.parse((String) generator.getNameValue().getValue()); - Map fieldWithType = generator.getFieldWithType(type); + Map fieldWithType = generator.getFieldWithElement(); Assert.assertTrue(fieldWithType.containsKey(BaseGenerator.ID)); } From da1207aee240487c22d48f7be869b1f76f2f586f Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 22 Oct 2021 18:27:22 +0400 Subject: [PATCH 055/405] DDP-7047 | move mapping to Util, add wrapper method for tests for db element --- .../dsm/model/elastic/Util.java | 20 ++++++++++++++++ .../model/elastic/export/BaseGenerator.java | 23 +++++++------------ .../model/elastic/export/ExportFacade.java | 19 ++++++++++++++- .../elastic/export/MappingGenerator.java | 11 ++------- .../model/elastic/export/SourceGenerator.java | 3 +-- .../dsm/statics/ESObjectConstants.java | 1 + 6 files changed, 50 insertions(+), 27 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 7fa4bfb46..6cab4ace4 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -1,9 +1,25 @@ package org.broadinstitute.dsm.model.elastic; +import java.util.Map; +import java.util.Objects; + +import org.broadinstitute.dsm.db.structure.DBElement; +import org.broadinstitute.dsm.model.elastic.export.BaseGenerator; +import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ParticipantUtil; +import org.broadinstitute.dsm.util.PatchUtil; public class Util { + public static final Map TABLE_ALIAS_MAPPINGS = Map.of( + "m", new BaseGenerator.PropertyInfo(ESObjectConstants.MEDICAL_RECORDS, true), + "t", new BaseGenerator.PropertyInfo(ESObjectConstants.TISSUE_RECORDS, true), + "oD", new BaseGenerator.PropertyInfo(ESObjectConstants.ONC_HISTORY_DETAIL_RECORDS, true), + "r", new BaseGenerator.PropertyInfo("participant", false), + "p", new BaseGenerator.PropertyInfo("participant", false), + "d", new BaseGenerator.PropertyInfo("participant", false) + ); + public static String getQueryTypeFromId(String id) { String type; if (ParticipantUtil.isHruid(id)) { @@ -17,4 +33,8 @@ public static String getQueryTypeFromId(String id) { } return type; } + + public static DBElement getDBElement(String fieldName) { + return PatchUtil.getColumnNameMap().get(Objects.requireNonNull(fieldName)); + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java index 1c35d7dad..7e6d9b2c4 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java @@ -4,7 +4,7 @@ import com.google.gson.JsonSyntaxException; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; -import org.broadinstitute.dsm.util.PatchUtil; +import org.broadinstitute.dsm.model.elastic.Util; import java.util.Map; import java.util.Objects; @@ -14,14 +14,6 @@ public abstract class BaseGenerator implements Generator { public static final String DSM_OBJECT = "dsm"; public static final String PROPERTIES = "properties"; - public static final Map TABLE_ALIAS_MAPPINGS = Map.of( - "m", new PropertyInfo("medicalRecords", true), - "t", new PropertyInfo("tissueRecords", true), - "oD", new PropertyInfo("oncHistoryDetailRecords", true), - "r", new PropertyInfo("participant", false), - "p", new PropertyInfo("participant", false), - "d", new PropertyInfo("participant", false) - ); public static final String ID = "id"; protected static final Gson GSON = new Gson(); protected final Parser parser; @@ -32,16 +24,17 @@ public BaseGenerator(Parser parser, GeneratorPayload generatorPayload) { this.generatorPayload = Objects.requireNonNull(generatorPayload); } - protected DBElement getDBElement() { - return PatchUtil.getColumnNameMap().get(Objects.requireNonNull(getNameValue()).getName()); - } - protected NameValue getNameValue() { return generatorPayload.getNameValue(); } - + + //wrap Util.getDBElement in protected method so that we can override it in testing class for tests + protected DBElement getDBElement() { + return Util.getDBElement(getNameValue().getName()); + } + protected PropertyInfo getOuterPropertyByAlias() { - return TABLE_ALIAS_MAPPINGS.get(getDBElement().getTableAlias()); + return Util.TABLE_ALIAS_MAPPINGS.get(getDBElement().getTableAlias()); } protected Map collect() { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 19248c919..b574db997 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -1,12 +1,17 @@ package org.broadinstitute.dsm.model.elastic.export; +import java.lang.reflect.Field; +import java.util.Arrays; import java.util.Map; import java.util.Objects; +import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.elastic.ESDsm; +import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchable; +import org.broadinstitute.dsm.util.PatchUtil; public class ExportFacade { @@ -37,7 +42,19 @@ private void upsertMapping() { private void fetchData() { ElasticSearchParticipantDto participantById = searchable.getParticipantById(exportFacadePayload.getIndex(), exportFacadePayload.getDocId()); - participantById.getDsm().map(ESDsm::getMedicalRecords) + DBElement dbElement = PatchUtil.getColumnNameMap().get(exportFacadePayload.getGeneratorPayload().getNameValue().getName()); + BaseGenerator.PropertyInfo propertyInfo = Util.TABLE_ALIAS_MAPPINGS.get(dbElement.getTableAlias()); + Field field = + Arrays.stream(ESDsm.class.getDeclaredFields()).filter(f -> propertyInfo.propertyName.equals(f.getName())).findFirst().get(); + participantById.getDsm().map(esDsm -> { + Object o = null; + try { + o = field.get(esDsm); + } catch (IllegalAccessException e) { + e.printStackTrace(); + } + return o; + }) .ifPresent(medicalRecords -> { medicalRecords.stream().map() }); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java index 028cad1e3..7fc9116b1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java @@ -3,6 +3,8 @@ import java.util.HashMap; import java.util.Map; +import org.broadinstitute.dsm.model.elastic.Util; + public class MappingGenerator extends BaseGenerator { @@ -68,13 +70,4 @@ protected Map getElement(Object type) { return Map.of(getDBElement().getColumnName(), Map.of(MappingGenerator.TYPE, type)); } - protected Map getFieldWithType(Object type) { - return getOuterPropertyByAlias().isCollection() - ? Map.of( - ID, Map.of(TYPE, TYPE_KEYWORD), - getDBElement().getColumnName(), Map.of(MappingGenerator.TYPE, type) - ) - : Map.of(getDBElement().getColumnName(), Map.of(MappingGenerator.TYPE, type)); - } - } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java index aa6633e72..205a0037a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java @@ -1,10 +1,9 @@ package org.broadinstitute.dsm.model.elastic.export; -import org.broadinstitute.dsm.model.NameValue; +import org.broadinstitute.dsm.model.elastic.Util; import java.util.HashMap; import java.util.Map; -import java.util.Objects; public class SourceGenerator extends BaseGenerator { diff --git a/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java b/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java index cc753aa42..5269afcea 100644 --- a/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java +++ b/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java @@ -62,4 +62,5 @@ public class ESObjectConstants { public static final String DIAGNOSIS_MONTH = "diagnosisMonth"; public static final String HAS_CONSENTED_TO_TISSUE_SAMPLE = "hasConsentedToTissueSample"; public static final String DIAGNOSIS_YEAR = "diagnosisYear"; + public static final String ONC_HISTORY_DETAIL_RECORDS = "oncHistoryDetailRecords"; } From 7cfd15c61f96958f24a628123a028d94afcc63eb Mon Sep 17 00:00:00 2001 From: Nika Date: Fri, 22 Oct 2021 20:06:11 +0400 Subject: [PATCH 056/405] DDP-7047 | introduce Processor interface and CollectionProcessor concrete class which will update values for nested data types for ES --- .../elastic/export/CollectionProcessor.java | 61 +++++++++++++++++++ .../model/elastic/export/ExportFacade.java | 10 +-- .../dsm/model/elastic/export/Processor.java | 8 +++ 3 files changed, 74 insertions(+), 5 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/Processor.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java new file mode 100644 index 000000000..474ae81d8 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java @@ -0,0 +1,61 @@ +package org.broadinstitute.dsm.model.elastic.export; + +import org.broadinstitute.dsm.model.elastic.ESDsm; + +import java.lang.reflect.Field; +import java.util.*; +import java.util.function.Predicate; + +public class CollectionProcessor implements Processor { + + private ESDsm esDsm; + private String propertyName; + private GeneratorPayload generatorPayload; + + private final Predicate isFieldMatchProperty = field -> propertyName.equals(field.getName()); + + public CollectionProcessor(ESDsm esDsm, String propertyName, GeneratorPayload generatorPayload) { + this.esDsm = Objects.requireNonNull(esDsm); + this.propertyName = Objects.requireNonNull(propertyName); + this.generatorPayload = Objects.requireNonNull(generatorPayload); + } + + @Override + public List> process() { + // medicalRecords: + // { + // id: 1 + // name: a + // } + // { + // id: 2 + // name: a (SSADASD) + // } + // { + // id: 3 + // name: b + // } + Field[] declaredFields = esDsm.getClass().getDeclaredFields(); + Field field = Arrays.stream(declaredFields).filter(isFieldMatchProperty) + .findFirst() + .orElseThrow(); + List> data = Collections.emptyList(); + try { + data = (List>) field.get(esDsm); + } catch (IllegalAccessException e) { + e.printStackTrace(); + } + + long recordId = generatorPayload.getRecordId(); + + + for (Map datum : data) { + //damekarge premier? me mesmis sheni me ki kai wamo aq gavtishot xma da + if ((long) datum.get(BaseGenerator.ID) == recordId) { + datum.put(propertyName, generatorPayload.getNameValue().getValue()); + } + } + + return null; + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index b574db997..002789f3c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -2,6 +2,7 @@ import java.lang.reflect.Field; import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.Objects; @@ -44,20 +45,19 @@ private void fetchData() { ElasticSearchParticipantDto participantById = searchable.getParticipantById(exportFacadePayload.getIndex(), exportFacadePayload.getDocId()); DBElement dbElement = PatchUtil.getColumnNameMap().get(exportFacadePayload.getGeneratorPayload().getNameValue().getName()); BaseGenerator.PropertyInfo propertyInfo = Util.TABLE_ALIAS_MAPPINGS.get(dbElement.getTableAlias()); + Field field = Arrays.stream(ESDsm.class.getDeclaredFields()).filter(f -> propertyInfo.propertyName.equals(f.getName())).findFirst().get(); participantById.getDsm().map(esDsm -> { - Object o = null; + List> o = null; try { - o = field.get(esDsm); + o = (List>) field.get(esDsm); } catch (IllegalAccessException e) { e.printStackTrace(); } return o; - }) - .ifPresent(medicalRecords -> { - medicalRecords.stream().map() }); + } private void upsertData() { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Processor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/Processor.java new file mode 100644 index 000000000..fdd97ea1b --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/Processor.java @@ -0,0 +1,8 @@ +package org.broadinstitute.dsm.model.elastic.export; + +import java.util.List; +import java.util.Map; + +public interface Processor { + List> process(); +} From bfd73d9959624a24d05ee01b5c1d14214b0af1c7 Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 25 Oct 2021 10:41:04 +0400 Subject: [PATCH 057/405] DDP-7047 introduce test for CollectionProcessor.java in order to test process() method --- .../elastic/export/CollectionProcessor.java | 20 ++-------- .../export/CollectionProcessorTest.java | 40 +++++++++++++++++++ 2 files changed, 43 insertions(+), 17 deletions(-) create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessorTest.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java index 474ae81d8..8199c5c39 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java @@ -22,23 +22,11 @@ public CollectionProcessor(ESDsm esDsm, String propertyName, GeneratorPayload ge @Override public List> process() { - // medicalRecords: - // { - // id: 1 - // name: a - // } - // { - // id: 2 - // name: a (SSADASD) - // } - // { - // id: 3 - // name: b - // } Field[] declaredFields = esDsm.getClass().getDeclaredFields(); Field field = Arrays.stream(declaredFields).filter(isFieldMatchProperty) .findFirst() .orElseThrow(); + field.setAccessible(true); List> data = Collections.emptyList(); try { data = (List>) field.get(esDsm); @@ -48,14 +36,12 @@ public List> process() { long recordId = generatorPayload.getRecordId(); - for (Map datum : data) { - //damekarge premier? me mesmis sheni me ki kai wamo aq gavtishot xma da if ((long) datum.get(BaseGenerator.ID) == recordId) { - datum.put(propertyName, generatorPayload.getNameValue().getValue()); + datum.put(generatorPayload.getNameValue().getName(), generatorPayload.getNameValue().getValue()); } } - return null; + return data; } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessorTest.java new file mode 100644 index 000000000..56dda8e01 --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessorTest.java @@ -0,0 +1,40 @@ +package org.broadinstitute.dsm.model.elastic.export; + +import com.google.gson.Gson; +import org.broadinstitute.dsm.model.NameValue; +import org.broadinstitute.dsm.model.elastic.ESDsm; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; +import java.util.Map; + +public class CollectionProcessorTest { + + @Test + public void testProcess() { + + String json = "{\"medicalRecords\":[{\"id\":10000000000000000,\"mr\":\"mr old\"}]}"; + ESDsm esDsm = new Gson().fromJson(json, ESDsm.class); + String propertyName = "medicalRecords"; + NameValue nameValue = new NameValue("mr", "mr updated"); + long recordId = 5; + GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, recordId); + + CollectionProcessor collectionProcessor = new CollectionProcessor(esDsm, propertyName, generatorPayload); + + List> process = collectionProcessor.process(); + + Map updatedObject = process.get(0); + Map oldObject = new Gson().fromJson(json, Map.class); + + Assert.assertEquals(oldObject, updatedObject); + + } + + private static class MedicalRecord { + + + } +} + From cdd60e94ce51e7afe5901342e91e9a2afcf25471 Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 25 Oct 2021 15:31:01 +0400 Subject: [PATCH 058/405] DDP-7047 | refactor process() method in CollectionProcessor --- .../elastic/export/CollectionProcessor.java | 38 +++++++++++++------ .../model/elastic/export/ExportFacade.java | 2 + .../elastic/export/GeneratorPayload.java | 6 +-- .../export/CollectionProcessorTest.java | 30 +++++++-------- .../elastic/export/MappingGeneratorTest.java | 12 +++--- .../elastic/export/SourceGeneratorTest.java | 6 +-- 6 files changed, 56 insertions(+), 38 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java index 8199c5c39..d920c81d1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java @@ -5,6 +5,7 @@ import java.lang.reflect.Field; import java.util.*; import java.util.function.Predicate; +import java.util.stream.Collectors; public class CollectionProcessor implements Processor { @@ -22,26 +23,41 @@ public CollectionProcessor(ESDsm esDsm, String propertyName, GeneratorPayload ge @Override public List> process() { + List> fetchedRecords = extractDataByReflection(); + return updateIfExists(fetchedRecords); + } + + private List> extractDataByReflection() { Field[] declaredFields = esDsm.getClass().getDeclaredFields(); Field field = Arrays.stream(declaredFields).filter(isFieldMatchProperty) .findFirst() .orElseThrow(); field.setAccessible(true); - List> data = Collections.emptyList(); + List> fetchedRecords; try { - data = (List>) field.get(esDsm); - } catch (IllegalAccessException e) { - e.printStackTrace(); + fetchedRecords = (List>) field.get(esDsm); + } catch (IllegalAccessException iae) { + throw new RuntimeException("error occurred while attempting to get data from ESDsm", iae); } + if (fetchedRecords == null) { + fetchedRecords = Collections.emptyList(); + } + return fetchedRecords; + } - long recordId = generatorPayload.getRecordId(); + private List> updateIfExists(List> fetchedRecords) { + return fetchedRecords.stream() + .filter(this::isExistingRecord) + .map(this::updateExistingRecord) + .collect(Collectors.toList()); + } - for (Map datum : data) { - if ((long) datum.get(BaseGenerator.ID) == recordId) { - datum.put(generatorPayload.getNameValue().getName(), generatorPayload.getNameValue().getValue()); - } - } + private boolean isExistingRecord(Map eachRecord) { + return (int) eachRecord.get(BaseGenerator.ID) == generatorPayload.getRecordId(); + } - return data; + private Map updateExistingRecord(Map eachRecord) { + eachRecord.put(generatorPayload.getNameValue().getName(), generatorPayload.getNameValue().getValue()); + return eachRecord; } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 002789f3c..06a24f626 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -20,11 +20,13 @@ public class ExportFacade { Generator generator; ElasticSearchable searchable; private ExportFacadePayload exportFacadePayload; + Processor processor; public ExportFacade(ExportFacadePayload exportFacadePayload) { this.exportFacadePayload = Objects.requireNonNull(exportFacadePayload); exportable = new ElasticExportAdapter(); searchable = new ElasticSearch(); + } public void export() { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/GeneratorPayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/GeneratorPayload.java index 6400a2662..81438c71e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/GeneratorPayload.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/GeneratorPayload.java @@ -5,9 +5,9 @@ public class GeneratorPayload { NameValue nameValue; - long recordId; + int recordId; - public GeneratorPayload(NameValue nameValue, long recordId) { + public GeneratorPayload(NameValue nameValue, int recordId) { this.nameValue = nameValue; this.recordId = recordId; } @@ -16,7 +16,7 @@ public NameValue getNameValue() { return nameValue; } - public long getRecordId() { + public int getRecordId() { return recordId; } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessorTest.java index 56dda8e01..fa1447eda 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessorTest.java @@ -1,39 +1,39 @@ package org.broadinstitute.dsm.model.elastic.export; -import com.google.gson.Gson; +import com.fasterxml.jackson.databind.ObjectMapper; import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.elastic.ESDsm; import org.junit.Assert; import org.junit.Test; +import java.io.IOException; import java.util.List; import java.util.Map; public class CollectionProcessorTest { @Test - public void testProcess() { - - String json = "{\"medicalRecords\":[{\"id\":10000000000000000,\"mr\":\"mr old\"}]}"; - ESDsm esDsm = new Gson().fromJson(json, ESDsm.class); + public void testProcess() throws IOException { String propertyName = "medicalRecords"; - NameValue nameValue = new NameValue("mr", "mr updated"); - long recordId = 5; - GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, recordId); + int recordId = 5; + String oldValue = "mr_old"; + String json = String.format("{\"%s\":[{\"id\":%s,\"mr\":\"%s\"}]}", propertyName, recordId, oldValue); - CollectionProcessor collectionProcessor = new CollectionProcessor(esDsm, propertyName, generatorPayload); + ObjectMapper objectMapper = new ObjectMapper(); - List> process = collectionProcessor.process(); + ESDsm esDsm = objectMapper.readValue(json, ESDsm.class); - Map updatedObject = process.get(0); - Map oldObject = new Gson().fromJson(json, Map.class); + NameValue nameValue = new NameValue("mr", "mr_updated"); + GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, recordId); - Assert.assertEquals(oldObject, updatedObject); + CollectionProcessor collectionProcessor = new CollectionProcessor(esDsm, propertyName, generatorPayload); - } + List> updatedList = collectionProcessor.process(); - private static class MedicalRecord { + Map updatedObject = updatedList.get(0); + Map oldObject = ((Map) ((List) objectMapper.readValue(json, Map.class).get(propertyName)).get(0)); + Assert.assertNotEquals(oldObject, updatedObject); } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java index 879feaa7f..7ebaf6488 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java @@ -15,7 +15,7 @@ public class MappingGeneratorTest { public void generateTextType() { GeneratorPayload generatorPayload = new GeneratorPayload( new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "value"), - 0L + 0 ); Map objectMap = TestMappingGenerator.of(generatorPayload).generate(); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), BaseGenerator.PROPERTIES); @@ -27,7 +27,7 @@ public void generateTextType() { public void generateBooleanType() { GeneratorPayload generatorPayload = new GeneratorPayload( new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "true"), - 0L + 0 ); Map objectMap = TestMappingGenerator.of(generatorPayload).generate(); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), BaseGenerator.PROPERTIES); @@ -39,7 +39,7 @@ public void generateBooleanType() { public void generateIntegerType() { GeneratorPayload generatorPayload = new GeneratorPayload( new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "45"), - 0L + 0 ); Map objectMap = TestMappingGenerator.of(generatorPayload).generate(); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), BaseGenerator.PROPERTIES); @@ -51,7 +51,7 @@ public void generateIntegerType() { public void generateDateType() { GeneratorPayload generatorPayload = new GeneratorPayload( new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "2021-10-30"), - 0L + 0 ); Map objectMap = TestMappingGenerator.of(generatorPayload).generate(); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), BaseGenerator.PROPERTIES); @@ -63,7 +63,7 @@ public void generateDateType() { public void generateNestedType() { GeneratorPayload generatorPayload = new GeneratorPayload( new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "2021-10-30"), - 100L + 100 ); Map objectMap = TestMappingGenerator.of(generatorPayload).generate(); Assert.assertEquals(MappingGenerator.NESTED, getMedicalRecordProperty(objectMap).get(MappingGenerator.TYPE)); @@ -73,7 +73,7 @@ public void generateNestedType() { public void getFieldWithTypeCollectionTrue() { GeneratorPayload generatorPayload = new GeneratorPayload( new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "2021-10-30"), - 100L + 100 ); TestMappingGenerator generator = TestMappingGenerator.of(generatorPayload); generator.getOuterPropertyByAlias().isCollection = true; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java index c2f532e7e..9d5321371 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java @@ -12,7 +12,7 @@ public class SourceGeneratorTest { @Test public void generate() { NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "value"); - GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, 0L); + GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, 0); Generator generator = new TestSourceGenerator(new ValueParser(), generatorPayload); Map objectMap = generator.generate(); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), SourceGenerator.DSM_OBJECT); @@ -23,7 +23,7 @@ public void generate() { @Test public void generateNumeric() { NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "1"); - GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, 0L); + GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, 0); Generator generator = new TestSourceGenerator(new ValueParser(), generatorPayload); Map objectMap = generator.generate(); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), SourceGenerator.DSM_OBJECT); @@ -34,7 +34,7 @@ public void generateNumeric() { @Test public void generateFromJson() { NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "{\"DDP_INSTANCE\": \"TEST\"}"); - GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, 0L); + GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, 0); Generator generator = new TestSourceGenerator(new ValueParser(), generatorPayload); Map objectMap = generator.generate(); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), SourceGenerator.DSM_OBJECT); From c509be8d02bbaa37331f424fe707473c6e819647 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 25 Oct 2021 16:26:11 +0400 Subject: [PATCH 059/405] DDP-7047 | add test for empty json for processor --- .../model/elastic/export/ExportFacade.java | 15 +++--------- .../export/CollectionProcessorTest.java | 24 +++++++++++++++++++ 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 06a24f626..1aa003ad4 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -47,18 +47,9 @@ private void fetchData() { ElasticSearchParticipantDto participantById = searchable.getParticipantById(exportFacadePayload.getIndex(), exportFacadePayload.getDocId()); DBElement dbElement = PatchUtil.getColumnNameMap().get(exportFacadePayload.getGeneratorPayload().getNameValue().getName()); BaseGenerator.PropertyInfo propertyInfo = Util.TABLE_ALIAS_MAPPINGS.get(dbElement.getTableAlias()); - - Field field = - Arrays.stream(ESDsm.class.getDeclaredFields()).filter(f -> propertyInfo.propertyName.equals(f.getName())).findFirst().get(); - participantById.getDsm().map(esDsm -> { - List> o = null; - try { - o = (List>) field.get(esDsm); - } catch (IllegalAccessException e) { - e.printStackTrace(); - } - return o; - }); + ESDsm esDsm = participantById.getDsm().orElseThrow(); + processor = new CollectionProcessor(esDsm, propertyInfo.getPropertyName(), exportFacadePayload.getGeneratorPayload()); + List> processedData = processor.process(); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessorTest.java index fa1447eda..9ab4a045b 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessorTest.java @@ -36,5 +36,29 @@ public void testProcess() throws IOException { Assert.assertNotEquals(oldObject, updatedObject); } + + @Test + public void testProcessOnEmpty() throws IOException { + String propertyName = "medicalRecords"; + int recordId = 5; + String json = ""; + + ObjectMapper objectMapper = new ObjectMapper(); + + ESDsm esDsm = objectMapper.readValue(json, ESDsm.class); + + NameValue nameValue = new NameValue("mr", "mr_updated"); + GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, recordId); + + CollectionProcessor collectionProcessor = new CollectionProcessor(esDsm, propertyName, generatorPayload); + + List> updatedList = collectionProcessor.process(); + + Map updatedObject = updatedList.get(0); + Map oldObject = ((Map) ((List) objectMapper.readValue(json, Map.class).get(propertyName)).get(0)); + + Assert.assertNotEquals(oldObject, updatedObject); + + } } From 7c150637e18428b7cf4307755d33779b89a7bf28 Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 25 Oct 2021 17:02:22 +0400 Subject: [PATCH 060/405] DDP-7047 | refactor ExportFacade.java based on conditional logic applied to returned list from fetchData() method --- .../model/elastic/export/ExportFacade.java | 30 +++++++++++-------- .../export/CollectionProcessorTest.java | 7 ++--- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 1aa003ad4..d4b5a5b55 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -1,7 +1,5 @@ package org.broadinstitute.dsm.model.elastic.export; -import java.lang.reflect.Field; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; @@ -26,13 +24,11 @@ public ExportFacade(ExportFacadePayload exportFacadePayload) { this.exportFacadePayload = Objects.requireNonNull(exportFacadePayload); exportable = new ElasticExportAdapter(); searchable = new ElasticSearch(); - } public void export() { upsertMapping(); - fetchData(); - upsertData(); + upsertData(processData(fetchData())); } private void upsertMapping() { @@ -43,19 +39,29 @@ private void upsertMapping() { exportable.exportMapping(mappingToUpsert); } - private void fetchData() { + private ESDsm fetchData() { ElasticSearchParticipantDto participantById = searchable.getParticipantById(exportFacadePayload.getIndex(), exportFacadePayload.getDocId()); - DBElement dbElement = PatchUtil.getColumnNameMap().get(exportFacadePayload.getGeneratorPayload().getNameValue().getName()); - BaseGenerator.PropertyInfo propertyInfo = Util.TABLE_ALIAS_MAPPINGS.get(dbElement.getTableAlias()); - ESDsm esDsm = participantById.getDsm().orElseThrow(); + return participantById.getDsm().orElseThrow(); + } + + private Map processData(ESDsm esDsm) { + BaseGenerator.PropertyInfo propertyInfo = getPropertyInfo(); processor = new CollectionProcessor(esDsm, propertyInfo.getPropertyName(), exportFacadePayload.getGeneratorPayload()); List> processedData = processor.process(); + Map dataToReturn = Map.of(propertyInfo.getPropertyName(), processedData); + if (processedData.isEmpty()) { + generator = new SourceGenerator(new ValueParser(), exportFacadePayload.getGeneratorPayload()); + dataToReturn = generator.generate(); + } + return dataToReturn; + } + private BaseGenerator.PropertyInfo getPropertyInfo() { + DBElement dbElement = PatchUtil.getColumnNameMap().get(exportFacadePayload.getGeneratorPayload().getNameValue().getName()); + return Util.TABLE_ALIAS_MAPPINGS.get(dbElement.getTableAlias()); } - private void upsertData() { - generator = new SourceGenerator(new ValueParser(), exportFacadePayload.getGeneratorPayload()); - Map elasticDataToExport = generator.generate(); + private void upsertData(Map elasticDataToExport) { UpsertDataRequestPayload upsertDataRequestPayload = new UpsertDataRequestPayload.Builder(exportFacadePayload.getIndex(), exportFacadePayload.getDocId()) .withDocAsUpsert(true) diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessorTest.java index 9ab4a045b..e56981c96 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessorTest.java @@ -41,7 +41,7 @@ public void testProcess() throws IOException { public void testProcessOnEmpty() throws IOException { String propertyName = "medicalRecords"; int recordId = 5; - String json = ""; + String json = "{}"; ObjectMapper objectMapper = new ObjectMapper(); @@ -54,10 +54,7 @@ public void testProcessOnEmpty() throws IOException { List> updatedList = collectionProcessor.process(); - Map updatedObject = updatedList.get(0); - Map oldObject = ((Map) ((List) objectMapper.readValue(json, Map.class).get(propertyName)).get(0)); - - Assert.assertNotEquals(oldObject, updatedObject); + Assert.assertTrue(updatedList.isEmpty()); } } From fb4455684c9202403f91cd333de70faa21d98b96 Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 26 Oct 2021 17:12:12 +0400 Subject: [PATCH 061/405] DDP-7047 | use CollectionProcessor.java where necessary --- .../elastic/export/CollectionProcessor.java | 20 ++++++++++--------- .../model/elastic/export/ExportFacade.java | 2 +- .../model/elastic/export/SourceGenerator.java | 3 +-- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java index d920c81d1..8c8905f17 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java @@ -29,20 +29,22 @@ public List> process() { private List> extractDataByReflection() { Field[] declaredFields = esDsm.getClass().getDeclaredFields(); - Field field = Arrays.stream(declaredFields).filter(isFieldMatchProperty) + List> fetchedRecords = Arrays.stream(declaredFields).filter(isFieldMatchProperty) .findFirst() - .orElseThrow(); - field.setAccessible(true); - List> fetchedRecords; + .map(field -> { + field.setAccessible(true); + return getRecordsByField(field); + }) + .orElse(Collections.emptyList()); + return fetchedRecords; + } + + private List> getRecordsByField(Field field) { try { - fetchedRecords = (List>) field.get(esDsm); + return (List>) field.get(esDsm); } catch (IllegalAccessException iae) { throw new RuntimeException("error occurred while attempting to get data from ESDsm", iae); } - if (fetchedRecords == null) { - fetchedRecords = Collections.emptyList(); - } - return fetchedRecords; } private List> updateIfExists(List> fetchedRecords) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index d4b5a5b55..2f5a7ac57 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -49,7 +49,7 @@ private Map processData(ESDsm esDsm) { processor = new CollectionProcessor(esDsm, propertyInfo.getPropertyName(), exportFacadePayload.getGeneratorPayload()); List> processedData = processor.process(); Map dataToReturn = Map.of(propertyInfo.getPropertyName(), processedData); - if (processedData.isEmpty()) { + if (propertyInfo.isCollection() && processedData.isEmpty()) { generator = new SourceGenerator(new ValueParser(), exportFacadePayload.getGeneratorPayload()); dataToReturn = generator.generate(); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java index 205a0037a..10359f842 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java @@ -38,8 +38,7 @@ protected Map parseSingleElement() { protected Map getElementWithId(Object element) { return Map.of( getOuterPropertyByAlias().getPropertyName(), Map.of(getDBElement().getColumnName(), - element), - ID, generatorPayload.getRecordId()); + element, ID, generatorPayload.getRecordId())); } @Override From ff97facbb27416eb0349fc17ad91cef54bc4c5ab Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 27 Oct 2021 11:14:29 +0400 Subject: [PATCH 062/405] DDP-7047 | fix nested objects bug, add test for mapping --- .../dsm/model/elastic/ESDsm.java | 3 +++ .../elastic/export/CollectionProcessor.java | 27 ++++++++++++------- .../model/elastic/export/ExportFacade.java | 2 +- .../model/elastic/export/SourceGenerator.java | 5 ++-- .../dsm/model/patch/TissuePatch.java | 2 ++ .../export/CollectionProcessorTest.java | 14 +++++----- .../elastic/export/MappingGeneratorTest.java | 19 +++++++++++++ 7 files changed, 52 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java index c9465c80a..ffde4b11f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java @@ -40,4 +40,7 @@ public class ESDsm { @SerializedName(ESObjectConstants.MEDICAL_RECORDS) List> medicalRecords; + @SerializedName(ESObjectConstants.ONC_HISTORY_DETAIL_RECORDS) + List> oncHistoryDetailRecords; + } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java index 8c8905f17..862e745ad 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java @@ -4,8 +4,8 @@ import java.lang.reflect.Field; import java.util.*; +import java.util.function.Function; import java.util.function.Predicate; -import java.util.stream.Collectors; public class CollectionProcessor implements Processor { @@ -24,7 +24,7 @@ public CollectionProcessor(ESDsm esDsm, String propertyName, GeneratorPayload ge @Override public List> process() { List> fetchedRecords = extractDataByReflection(); - return updateIfExists(fetchedRecords); + return updateIfExistsOrPut(fetchedRecords); } private List> extractDataByReflection() { @@ -35,7 +35,7 @@ private List> extractDataByReflection() { field.setAccessible(true); return getRecordsByField(field); }) - .orElse(Collections.emptyList()); + .orElse(new ArrayList<>()); return fetchedRecords; } @@ -47,19 +47,26 @@ private List> getRecordsByField(Field field) { } } - private List> updateIfExists(List> fetchedRecords) { - return fetchedRecords.stream() + private List> updateIfExistsOrPut(List> fetchedRecords) { + fetchedRecords.stream() .filter(this::isExistingRecord) - .map(this::updateExistingRecord) - .collect(Collectors.toList()); + .findFirst() + .ifPresentOrElse(this::updateExistingRecord, () -> addNewRecordTo(fetchedRecords)); + return fetchedRecords; + } + + private void addNewRecordTo(List> fetchedRecords) { + fetchedRecords.add(Map.of( + MappingGenerator.ID, generatorPayload.getRecordId(), + generatorPayload.getNameValue().getName(), generatorPayload.getNameValue().getValue() + )); } private boolean isExistingRecord(Map eachRecord) { - return (int) eachRecord.get(BaseGenerator.ID) == generatorPayload.getRecordId(); + return (double) eachRecord.get(BaseGenerator.ID) == (double) generatorPayload.getRecordId(); } - private Map updateExistingRecord(Map eachRecord) { + private void updateExistingRecord(Map eachRecord) { eachRecord.put(generatorPayload.getNameValue().getName(), generatorPayload.getNameValue().getValue()); - return eachRecord; } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 2f5a7ac57..58349a467 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -48,7 +48,7 @@ private Map processData(ESDsm esDsm) { BaseGenerator.PropertyInfo propertyInfo = getPropertyInfo(); processor = new CollectionProcessor(esDsm, propertyInfo.getPropertyName(), exportFacadePayload.getGeneratorPayload()); List> processedData = processor.process(); - Map dataToReturn = Map.of(propertyInfo.getPropertyName(), processedData); + Map dataToReturn = Map.of(MappingGenerator.DSM_OBJECT, Map.of(propertyInfo.getPropertyName(), processedData)); if (propertyInfo.isCollection() && processedData.isEmpty()) { generator = new SourceGenerator(new ValueParser(), exportFacadePayload.getGeneratorPayload()); dataToReturn = generator.generate(); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java index 10359f842..922305b0d 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java @@ -3,6 +3,7 @@ import org.broadinstitute.dsm.model.elastic.Util; import java.util.HashMap; +import java.util.List; import java.util.Map; public class SourceGenerator extends BaseGenerator { @@ -37,8 +38,8 @@ protected Map parseSingleElement() { @Override protected Map getElementWithId(Object element) { return Map.of( - getOuterPropertyByAlias().getPropertyName(), Map.of(getDBElement().getColumnName(), - element, ID, generatorPayload.getRecordId())); + getOuterPropertyByAlias().getPropertyName(), List.of(Map.of(getDBElement().getColumnName(), + element, ID, generatorPayload.getRecordId()))); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java index 8fc2e42cb..82998fbb5 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java @@ -44,6 +44,8 @@ Object handleSingleNameValue() { if (!nameValues.isEmpty()) { resultMap.put(NAME_VALUE, GSON.toJson(nameValues)); } + patch.setId(tissueId); + exportToES(patch.getNameValue()); } return resultMap; } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessorTest.java index e56981c96..1642426a8 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessorTest.java @@ -15,7 +15,7 @@ public class CollectionProcessorTest { @Test public void testProcess() throws IOException { String propertyName = "medicalRecords"; - int recordId = 5; + double recordId = 5; String oldValue = "mr_old"; String json = String.format("{\"%s\":[{\"id\":%s,\"mr\":\"%s\"}]}", propertyName, recordId, oldValue); @@ -24,7 +24,7 @@ public void testProcess() throws IOException { ESDsm esDsm = objectMapper.readValue(json, ESDsm.class); NameValue nameValue = new NameValue("mr", "mr_updated"); - GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, recordId); + GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, (int)recordId); CollectionProcessor collectionProcessor = new CollectionProcessor(esDsm, propertyName, generatorPayload); @@ -38,23 +38,23 @@ public void testProcess() throws IOException { } @Test - public void testProcessOnEmpty() throws IOException { + public void updateIfExistsOrPut() throws IOException { String propertyName = "medicalRecords"; - int recordId = 5; - String json = "{}"; + double recordId = 5; + String json = String.format("{\"%s\":[{\"id\":%s,\"mr\":\"%s\"}]}", propertyName, recordId, "value");; ObjectMapper objectMapper = new ObjectMapper(); ESDsm esDsm = objectMapper.readValue(json, ESDsm.class); NameValue nameValue = new NameValue("mr", "mr_updated"); - GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, recordId); + GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, 10); CollectionProcessor collectionProcessor = new CollectionProcessor(esDsm, propertyName, generatorPayload); List> updatedList = collectionProcessor.process(); - Assert.assertTrue(updatedList.isEmpty()); + Assert.assertEquals(2, updatedList.size()); } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java index 7ebaf6488..a61334094 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java @@ -81,6 +81,25 @@ public void getFieldWithTypeCollectionTrue() { Assert.assertTrue(fieldWithType.containsKey(BaseGenerator.ID)); } + @Test + public void generateMapping() { + GeneratorPayload generatorPayload = new GeneratorPayload( + new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "2021-10-30"), + 100 + ); + MappingGenerator generator = new TestMappingGenerator(new TypeParser(), generatorPayload); + Map resultMap = generator.generate(); + Map dsmLevelProperty = Map.of(generator.getOuterPropertyByAlias().getPropertyName(), Map.of( + MappingGenerator.TYPE, MappingGenerator.NESTED, + MappingGenerator.PROPERTIES, Map.of(TestPatchUtil.MEDICAL_RECORD_COLUMN, Map.of(MappingGenerator.TYPE, "date"), + MappingGenerator.ID, Map.of(MappingGenerator.TYPE, MappingGenerator.TYPE_KEYWORD) + ))); + Map dsmLevelProperties = Map.of(MappingGenerator.PROPERTIES, dsmLevelProperty); + Map dsmLevel = Map.of(MappingGenerator.DSM_OBJECT, dsmLevelProperties); + Map topLevel = Map.of(MappingGenerator.PROPERTIES, dsmLevel); + Assert.assertTrue(topLevel.equals(resultMap)); + } + private String extractDeepestLeveleValue(Map objectMap) { return (String) ((Map) From b71bcf1b0f1ecf4673c5348b308cd7df772f161d Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 27 Oct 2021 18:00:09 +0400 Subject: [PATCH 063/405] DDP-7047 | add export() to OncHistoryDetailPatch.java --- src/main/java/org/broadinstitute/dsm/model/elastic/Util.java | 3 +-- .../broadinstitute/dsm/model/patch/OncHistoryDetailPatch.java | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 6cab4ace4..bfe09fda3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -16,8 +16,7 @@ public class Util { "t", new BaseGenerator.PropertyInfo(ESObjectConstants.TISSUE_RECORDS, true), "oD", new BaseGenerator.PropertyInfo(ESObjectConstants.ONC_HISTORY_DETAIL_RECORDS, true), "r", new BaseGenerator.PropertyInfo("participant", false), - "p", new BaseGenerator.PropertyInfo("participant", false), - "d", new BaseGenerator.PropertyInfo("participant", false) + "p", new BaseGenerator.PropertyInfo("participant", false) ); public static String getQueryTypeFromId(String id) { diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/OncHistoryDetailPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/OncHistoryDetailPatch.java index 7a20f162c..6fac50cfa 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/OncHistoryDetailPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/OncHistoryDetailPatch.java @@ -76,6 +76,8 @@ protected Object patchNameValuePair() { Object handleSingleNameValue() { if (Patch.patch(oncHistoryDetailId, patch.getUser(), patch.getNameValue(), dbElement)) { nameValues.addAll(setWorkflowRelatedFields(patch)); + patch.setId(mrID.toString()); + exportToES(patch.getNameValue()); //set oncHistoryDetails created if it is a oncHistoryDetails value without a ID, otherwise created should already be set if (dbElement.getTableName().equals(DBConstants.DDP_ONC_HISTORY_DETAIL)) { NameValue oncHistoryCreated = OncHistory.setOncHistoryCreated(patch.getParentId(), patch.getUser()); @@ -91,6 +93,8 @@ Object handleSingleNameValue() { @Override Optional processEachNameValue(NameValue nameValue) { Patch.patch(oncHistoryDetailId, patch.getUser(), nameValue, dbElement); + patch.setId(mrID.toString()); + exportToES(nameValue); return Optional.empty(); } From e3b73bfb6d66dece2bea738e1c7e930debf7e5a3 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 27 Oct 2021 18:01:25 +0400 Subject: [PATCH 064/405] DDP-7047 | fix bug of sending everything as list to elastic search --- .../org/broadinstitute/dsm/model/elastic/Util.java | 1 + .../dsm/model/elastic/export/BaseGenerator.java | 8 ++++++-- .../dsm/model/elastic/export/ExportFacade.java | 14 ++++++++------ .../dsm/model/elastic/export/MappingGenerator.java | 2 +- .../dsm/model/elastic/export/SourceGenerator.java | 2 +- 5 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 6cab4ace4..820d436f9 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -15,6 +15,7 @@ public class Util { "m", new BaseGenerator.PropertyInfo(ESObjectConstants.MEDICAL_RECORDS, true), "t", new BaseGenerator.PropertyInfo(ESObjectConstants.TISSUE_RECORDS, true), "oD", new BaseGenerator.PropertyInfo(ESObjectConstants.ONC_HISTORY_DETAIL_RECORDS, true), + "o", new BaseGenerator.PropertyInfo(ESObjectConstants.ONC_HISTORY_DETAIL_RECORDS, true), "r", new BaseGenerator.PropertyInfo("participant", false), "p", new BaseGenerator.PropertyInfo("participant", false), "d", new BaseGenerator.PropertyInfo("participant", false) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java index 7e6d9b2c4..ef380e36a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java @@ -48,12 +48,16 @@ protected Map collect() { } protected abstract Map parseJson(); - + + protected Map parseJsonToMapFromValue() { + return GSON.fromJson(String.valueOf(getNameValue().getValue()), Map.class); + } + protected abstract Map parseSingleElement(); protected Map getFieldWithElement() { Map fieldElementMap; - Object element = parser.parse((String) getNameValue().getValue()); + Object element = parser.parse(String.valueOf(getNameValue().getValue())); if (getOuterPropertyByAlias().isCollection) { fieldElementMap = getElementWithId(element); } else { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 58349a467..09b33b966 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -46,12 +46,14 @@ private ESDsm fetchData() { private Map processData(ESDsm esDsm) { BaseGenerator.PropertyInfo propertyInfo = getPropertyInfo(); - processor = new CollectionProcessor(esDsm, propertyInfo.getPropertyName(), exportFacadePayload.getGeneratorPayload()); - List> processedData = processor.process(); - Map dataToReturn = Map.of(MappingGenerator.DSM_OBJECT, Map.of(propertyInfo.getPropertyName(), processedData)); - if (propertyInfo.isCollection() && processedData.isEmpty()) { - generator = new SourceGenerator(new ValueParser(), exportFacadePayload.getGeneratorPayload()); - dataToReturn = generator.generate(); + generator = new SourceGenerator(new ValueParser(), exportFacadePayload.getGeneratorPayload()); + Map dataToReturn = generator.generate(); + if (propertyInfo.isCollection()) { + processor = new CollectionProcessor(esDsm, propertyInfo.getPropertyName(), exportFacadePayload.getGeneratorPayload()); + List> processedData = processor.process(); + if (!processedData.isEmpty()) { + dataToReturn = Map.of(MappingGenerator.DSM_OBJECT, Map.of(propertyInfo.getPropertyName(), processedData)); + } } return dataToReturn; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java index 7fc9116b1..91a8d1332 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java @@ -44,7 +44,7 @@ private Map buildMappedField() { @Override protected Map parseJson() { Map resultMap = new HashMap<>(); - Map fieldsByValues = GSON.fromJson((String) getNameValue().getValue(), Map.class); + Map fieldsByValues = parseJsonToMapFromValue(); for (Map.Entry entry: fieldsByValues.entrySet()) { Object eachType = parser.parse((String) entry.getValue()); resultMap.put(entry.getKey(), Map.of(MappingGenerator.TYPE, eachType)); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java index 922305b0d..b1559082c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java @@ -22,7 +22,7 @@ public Map generate() { @Override protected Map parseJson() { - Map dynamicFieldValues = GSON.fromJson((String) getNameValue().getValue(), Map.class); + Map dynamicFieldValues = parseJsonToMapFromValue(); Map transformedMap = new HashMap<>(); for (Map.Entry entry : dynamicFieldValues.entrySet()) { transformedMap.put(entry.getKey(), parser.parse((String) entry.getValue())); From ff5ccbdbebb7ac003ae6dfb28fff6b4cecbe81d4 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 27 Oct 2021 18:35:26 +0400 Subject: [PATCH 065/405] DDP-7047 | fix class cast exception --- .../dsm/model/elastic/export/MappingGenerator.java | 2 +- .../dsm/model/elastic/export/SourceGenerator.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java index 91a8d1332..7641ef127 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java @@ -46,7 +46,7 @@ protected Map parseJson() { Map resultMap = new HashMap<>(); Map fieldsByValues = parseJsonToMapFromValue(); for (Map.Entry entry: fieldsByValues.entrySet()) { - Object eachType = parser.parse((String) entry.getValue()); + Object eachType = parser.parse(String.valueOf(entry.getValue())); resultMap.put(entry.getKey(), Map.of(MappingGenerator.TYPE, eachType)); } return resultMap; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java index b1559082c..41b5b952b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java @@ -25,7 +25,7 @@ protected Map parseJson() { Map dynamicFieldValues = parseJsonToMapFromValue(); Map transformedMap = new HashMap<>(); for (Map.Entry entry : dynamicFieldValues.entrySet()) { - transformedMap.put(entry.getKey(), parser.parse((String) entry.getValue())); + transformedMap.put(entry.getKey(), parser.parse(String.valueOf(entry.getValue()))); } return Map.of(getOuterPropertyByAlias().getPropertyName(), transformedMap); } From aef1081a0afa0a1dacad395c327f079c20f3c88c Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 28 Oct 2021 10:22:32 +0400 Subject: [PATCH 066/405] DDP-7047 | extract constant fields --- .../java/org/broadinstitute/dsm/model/elastic/Util.java | 9 +++++---- .../broadinstitute/dsm/statics/ESObjectConstants.java | 4 ++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 465400981..837a27afc 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -15,10 +15,11 @@ public class Util { "m", new BaseGenerator.PropertyInfo(ESObjectConstants.MEDICAL_RECORDS, true), "t", new BaseGenerator.PropertyInfo(ESObjectConstants.TISSUE_RECORDS, true), "oD", new BaseGenerator.PropertyInfo(ESObjectConstants.ONC_HISTORY_DETAIL_RECORDS, true), - "o", new BaseGenerator.PropertyInfo(ESObjectConstants.ONC_HISTORY_DETAIL_RECORDS, true), - "r", new BaseGenerator.PropertyInfo("participant", false), - "p", new BaseGenerator.PropertyInfo("participant", false) - ); + "d", new BaseGenerator.PropertyInfo(ESObjectConstants.PARTICIPANT_DATA, true), + "r", new BaseGenerator.PropertyInfo(ESObjectConstants.PARTICIPANT_RECORD, false), + "p", new BaseGenerator.PropertyInfo(ESObjectConstants.PARTICIPANT, false), + "o", new BaseGenerator.PropertyInfo(ESObjectConstants.ONC_HISTORY, false) + ); public static String getQueryTypeFromId(String id) { String type; diff --git a/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java b/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java index 5269afcea..3a37eca76 100644 --- a/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java +++ b/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java @@ -63,4 +63,8 @@ public class ESObjectConstants { public static final String HAS_CONSENTED_TO_TISSUE_SAMPLE = "hasConsentedToTissueSample"; public static final String DIAGNOSIS_YEAR = "diagnosisYear"; public static final String ONC_HISTORY_DETAIL_RECORDS = "oncHistoryDetailRecords"; + public static final String ONC_HISTORY = "oncHistory"; + public static final String PARTICIPANT_DATA = "participantData"; + public static final String PARTICIPANT_RECORD = "participantRecord"; + public static final String PARTICIPANT = "participant"; } From 4222f4a178bfdb0268c9aeb5638d703ab15d023d Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 28 Oct 2021 10:53:55 +0400 Subject: [PATCH 067/405] DDP-7047 | remove unused imports, add method to parse values which contain dot --- .../java/org/broadinstitute/dsm/model/elastic/Util.java | 7 +++++++ .../dsm/model/elastic/export/SourceGenerator.java | 2 -- .../dsm/model/elastic/export/TypeParser.java | 7 ------- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 837a27afc..78754f186 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -3,9 +3,11 @@ import java.util.Map; import java.util.Objects; +import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.elastic.export.BaseGenerator; import org.broadinstitute.dsm.statics.ESObjectConstants; +import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.broadinstitute.dsm.util.ParticipantUtil; import org.broadinstitute.dsm.util.PatchUtil; @@ -38,4 +40,9 @@ public static String getQueryTypeFromId(String id) { public static DBElement getDBElement(String fieldName) { return PatchUtil.getColumnNameMap().get(Objects.requireNonNull(fieldName)); } + + public static String parseDotValue(String value) { + String[] split = Objects.requireNonNull(value).split(ElasticSearchUtil.DOT_SEPARATOR); + return split.length > 1 ? split[1] : split[0]; + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java index 41b5b952b..c0c93e791 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java @@ -1,7 +1,5 @@ package org.broadinstitute.dsm.model.elastic.export; -import org.broadinstitute.dsm.model.elastic.Util; - import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/TypeParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/TypeParser.java index 4f0a8ac4f..231ea66e7 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/TypeParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/TypeParser.java @@ -1,12 +1,5 @@ package org.broadinstitute.dsm.model.elastic.export; -import org.apache.commons.lang3.StringUtils; - -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.LocalTime; -import java.time.format.DateTimeParseException; - public class TypeParser extends BaseParser { From c5330d31bf8845ff08d71ee6fe7d192f4ac32b8f Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 28 Oct 2021 16:57:01 +0400 Subject: [PATCH 068/405] DDP-7047 | initialize refactoring in order to fix bugs --- .../org/broadinstitute/dsm/model/elastic/Util.java | 5 ----- .../dsm/model/elastic/export/BaseGenerator.java | 12 ++++++------ .../model/elastic/export/CollectionProcessor.java | 2 +- .../dsm/model/elastic/export/MappingGenerator.java | 7 ++----- .../dsm/model/elastic/export/SourceGenerator.java | 9 ++++----- .../model/elastic/export/MappingGeneratorTest.java | 6 ++++-- 6 files changed, 17 insertions(+), 24 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 78754f186..96406ad99 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -40,9 +40,4 @@ public static String getQueryTypeFromId(String id) { public static DBElement getDBElement(String fieldName) { return PatchUtil.getColumnNameMap().get(Objects.requireNonNull(fieldName)); } - - public static String parseDotValue(String value) { - String[] split = Objects.requireNonNull(value).split(ElasticSearchUtil.DOT_SEPARATOR); - return split.length > 1 ? split[1] : split[0]; - } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java index ef380e36a..0646629e9 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java @@ -38,13 +38,13 @@ protected PropertyInfo getOuterPropertyByAlias() { } protected Map collect() { - Map sourceToUpsert; + Object sourceToUpsert; try { sourceToUpsert = parseJson(); } catch (JsonSyntaxException jse) { sourceToUpsert = parseSingleElement(); } - return sourceToUpsert; + return Map.of(getOuterPropertyByAlias().getPropertyName(), sourceToUpsert); } protected abstract Map parseJson(); @@ -53,10 +53,10 @@ protected Map parseJsonToMapFromValue() { return GSON.fromJson(String.valueOf(getNameValue().getValue()), Map.class); } - protected abstract Map parseSingleElement(); + protected abstract Object parseSingleElement(); - protected Map getFieldWithElement() { - Map fieldElementMap; + protected Object getFieldWithElement() { + Object fieldElementMap; Object element = parser.parse(String.valueOf(getNameValue().getValue())); if (getOuterPropertyByAlias().isCollection) { fieldElementMap = getElementWithId(element); @@ -66,7 +66,7 @@ protected Map getFieldWithElement() { return fieldElementMap; } - protected abstract Map getElementWithId(Object element); + protected abstract Object getElementWithId(Object element); protected abstract Map getElement(Object element); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java index 862e745ad..377894633 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java @@ -4,7 +4,6 @@ import java.lang.reflect.Field; import java.util.*; -import java.util.function.Function; import java.util.function.Predicate; public class CollectionProcessor implements Processor { @@ -24,6 +23,7 @@ public CollectionProcessor(ESDsm esDsm, String propertyName, GeneratorPayload ge @Override public List> process() { List> fetchedRecords = extractDataByReflection(); + Map.of(propertyName, fetchedRecords); return updateIfExistsOrPut(fetchedRecords); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java index 7641ef127..e0fa654b1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java @@ -19,11 +19,8 @@ public MappingGenerator(Parser parser, GeneratorPayload generatorPayload) { @Override public Map generate() { - PropertyInfo propertyInfo = getOuterPropertyByAlias(); - String propertyName = propertyInfo.getPropertyName(); Map mappedField = buildMappedField(); - Map dsmLevelProperty = Map.of(propertyName, mappedField); - Map dsmLevelProperties = Map.of(PROPERTIES, dsmLevelProperty); + Map dsmLevelProperties = Map.of(PROPERTIES, mappedField); Map> dsmLevel = Map.of(DSM_OBJECT, dsmLevelProperties); return Map.of(PROPERTIES, dsmLevel); } @@ -53,7 +50,7 @@ protected Map parseJson() { } @Override - protected Map parseSingleElement() { + protected Object parseSingleElement() { return getFieldWithElement(); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java index c0c93e791..fddd18fbf 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java @@ -29,15 +29,14 @@ protected Map parseJson() { } @Override - protected Map parseSingleElement() { + protected Object parseSingleElement() { return getFieldWithElement(); } @Override - protected Map getElementWithId(Object element) { - return Map.of( - getOuterPropertyByAlias().getPropertyName(), List.of(Map.of(getDBElement().getColumnName(), - element, ID, generatorPayload.getRecordId()))); + protected Object getElementWithId(Object element) { + return List.of(Map.of(getDBElement().getColumnName(),element, + ID, generatorPayload.getRecordId())); } @Override diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java index a61334094..8e0cf4adc 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java @@ -77,8 +77,8 @@ public void getFieldWithTypeCollectionTrue() { ); TestMappingGenerator generator = TestMappingGenerator.of(generatorPayload); generator.getOuterPropertyByAlias().isCollection = true; - Map fieldWithType = generator.getFieldWithElement(); - Assert.assertTrue(fieldWithType.containsKey(BaseGenerator.ID)); + Object fieldWithType = generator.getFieldWithElement(); + Assert.assertTrue(((Map) fieldWithType).containsKey(BaseGenerator.ID)); } @Test @@ -112,12 +112,14 @@ private String extractDeepestLeveleValue(Map objectMap) { private Map getMedicalRecordProperty(Map objectMap) { return (Map) + ((Map) ((Map) ((Map) ((Map) objectMap .get(BaseGenerator.PROPERTIES)) .get(BaseGenerator.DSM_OBJECT)) .get(BaseGenerator.PROPERTIES)) + .get(BaseGenerator.PROPERTIES)) .get("medicalRecords"); } From c46bb3e66042ced31c3d054ea12a2a504dc340d7 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 28 Oct 2021 18:28:28 +0400 Subject: [PATCH 069/405] DDP-7047 | change functionality for list sources, change tests --- .../model/elastic/export/BaseGenerator.java | 10 +++--- .../elastic/export/MappingGenerator.java | 6 ++-- .../model/elastic/export/SourceGenerator.java | 18 +++++++--- .../elastic/export/MappingGeneratorTest.java | 2 -- .../elastic/export/SourceGeneratorTest.java | 34 ++++++++++++------- .../model/elastic/export/TestPatchUtil.java | 7 ++-- 6 files changed, 50 insertions(+), 27 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java index 0646629e9..aaa469d3a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java @@ -37,23 +37,23 @@ protected PropertyInfo getOuterPropertyByAlias() { return Util.TABLE_ALIAS_MAPPINGS.get(getDBElement().getTableAlias()); } - protected Map collect() { + protected Object collect() { Object sourceToUpsert; try { sourceToUpsert = parseJson(); } catch (JsonSyntaxException jse) { sourceToUpsert = parseSingleElement(); } - return Map.of(getOuterPropertyByAlias().getPropertyName(), sourceToUpsert); + return sourceToUpsert; } - protected abstract Map parseJson(); + protected abstract T parseJson(); - protected Map parseJsonToMapFromValue() { + protected Map parseJsonToMapFromValue() { return GSON.fromJson(String.valueOf(getNameValue().getValue()), Map.class); } - protected abstract Object parseSingleElement(); + protected abstract T parseSingleElement(); protected Object getFieldWithElement() { Object fieldElementMap; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java index e0fa654b1..99c912535 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java @@ -19,8 +19,10 @@ public MappingGenerator(Parser parser, GeneratorPayload generatorPayload) { @Override public Map generate() { + String propertyName = getOuterPropertyByAlias().getPropertyName(); Map mappedField = buildMappedField(); - Map dsmLevelProperties = Map.of(PROPERTIES, mappedField); + Map objectLevel = Map.of(propertyName, mappedField); + Map dsmLevelProperties = Map.of(PROPERTIES, objectLevel); Map> dsmLevel = Map.of(DSM_OBJECT, dsmLevelProperties); return Map.of(PROPERTIES, dsmLevel); } @@ -28,7 +30,7 @@ public Map generate() { private Map buildMappedField() { PropertyInfo propertyInfo = getOuterPropertyByAlias(); boolean isPropertyCollection = propertyInfo.isCollection(); - Map field = collect(); + Object field = collect(); Map mappedField; if (isPropertyCollection) { mappedField = Map.of(TYPE, NESTED, PROPERTIES, field); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java index fddd18fbf..5db0f643a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java @@ -4,6 +4,8 @@ import java.util.List; import java.util.Map; +import org.jcodings.util.Hash; + public class SourceGenerator extends BaseGenerator { @@ -14,18 +16,26 @@ public SourceGenerator(Parser parser, GeneratorPayload generatorPayload) { @Override public Map generate() { - Map dataToExport = collect(); - return Map.of(DSM_OBJECT, dataToExport); + Object dataToExport = collect(); + Map objectLevel = Map.of(getOuterPropertyByAlias().getPropertyName(), dataToExport); + return Map.of(DSM_OBJECT, objectLevel); } @Override - protected Map parseJson() { + protected Object parseJson() { Map dynamicFieldValues = parseJsonToMapFromValue(); Map transformedMap = new HashMap<>(); for (Map.Entry entry : dynamicFieldValues.entrySet()) { transformedMap.put(entry.getKey(), parser.parse(String.valueOf(entry.getValue()))); } - return Map.of(getOuterPropertyByAlias().getPropertyName(), transformedMap); + if (getOuterPropertyByAlias().isCollection()) { + Map collectionMap = new HashMap<>(); + collectionMap.put(ID, generatorPayload.getRecordId()); + collectionMap.putAll(transformedMap); + return List.of(collectionMap); + } else { + return transformedMap; + } } @Override diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java index 8e0cf4adc..2fea35cce 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java @@ -112,14 +112,12 @@ private String extractDeepestLeveleValue(Map objectMap) { private Map getMedicalRecordProperty(Map objectMap) { return (Map) - ((Map) ((Map) ((Map) ((Map) objectMap .get(BaseGenerator.PROPERTIES)) .get(BaseGenerator.DSM_OBJECT)) .get(BaseGenerator.PROPERTIES)) - .get(BaseGenerator.PROPERTIES)) .get("medicalRecords"); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java index 9d5321371..006e66117 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java @@ -1,6 +1,8 @@ package org.broadinstitute.dsm.model.elastic.export; +import java.util.List; import java.util.Map; +import java.util.Optional; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; @@ -10,14 +12,17 @@ public class SourceGeneratorTest { @Test - public void generate() { + public void generateCollection() { NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "value"); GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, 0); Generator generator = new TestSourceGenerator(new ValueParser(), generatorPayload); Map objectMap = generator.generate(); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), SourceGenerator.DSM_OBJECT); - String value = (String)((Map)((Map)objectMap.get(SourceGenerator.DSM_OBJECT)).get("medicalRecords")).get(TestPatchUtil.MEDICAL_RECORD_COLUMN); - Assert.assertEquals("value", value); + List> medicalRecords = (List) ((Map) objectMap.get(SourceGenerator.DSM_OBJECT)).get("medicalRecords"); + Optional> first = medicalRecords.stream() + .filter(i -> i.get(TestPatchUtil.MEDICAL_RECORD_COLUMN) != null) + .findFirst(); + first.ifPresentOrElse(val -> Assert.assertEquals("value", val.get(TestPatchUtil.MEDICAL_RECORD_COLUMN)), Assert::fail); } @Test @@ -27,23 +32,28 @@ public void generateNumeric() { Generator generator = new TestSourceGenerator(new ValueParser(), generatorPayload); Map objectMap = generator.generate(); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), SourceGenerator.DSM_OBJECT); - Object value = ((Map)((Map)objectMap.get(SourceGenerator.DSM_OBJECT)).get("medicalRecords")).get(TestPatchUtil.MEDICAL_RECORD_COLUMN); - Assert.assertEquals(1L, value); + List> medicalRecords = (List) ((Map) objectMap.get(SourceGenerator.DSM_OBJECT)).get("medicalRecords"); + Optional> first = medicalRecords.stream() + .filter(i -> i.get(TestPatchUtil.MEDICAL_RECORD_COLUMN) != null) + .findFirst(); + first.ifPresentOrElse(val -> Assert.assertEquals(1L, val.get(TestPatchUtil.MEDICAL_RECORD_COLUMN)), Assert::fail); } @Test public void generateFromJson() { - NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "{\"DDP_INSTANCE\": \"TEST\"}"); + NameValue nameValue = new NameValue(TestPatchUtil.TISSUE_RECORD_COLUMN, "{\"DDP_INSTANCE\": \"TEST\", \"DPP_VALUE\": \"VALUE\"}"); GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, 0); Generator generator = new TestSourceGenerator(new ValueParser(), generatorPayload); Map objectMap = generator.generate(); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), SourceGenerator.DSM_OBJECT); - Object value = - ((Map)((Map)objectMap - .get(SourceGenerator.DSM_OBJECT)) - .get("medicalRecords")) - .get("DDP_INSTANCE"); - Assert.assertEquals("TEST", value); + + List> tissueRecords = (List>) ((Map) objectMap + .get(SourceGenerator.DSM_OBJECT)) + .get("tissueRecords"); + Optional> maybeDdpInstance = tissueRecords.stream() + .filter(i -> i.get("DDP_INSTANCE") != null) + .findFirst(); + maybeDdpInstance.ifPresentOrElse(m -> Assert.assertEquals("TEST", m.get("DDP_INSTANCE")), Assert::fail); } private static class TestSourceGenerator extends SourceGenerator { diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/TestPatchUtil.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/TestPatchUtil.java index dc9172440..7c17007a6 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/TestPatchUtil.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/TestPatchUtil.java @@ -8,11 +8,14 @@ class TestPatchUtil extends PatchUtil { public static final String MEDICAL_RECORD_COLUMN = "medical_record_column"; + public static final String TISSUE_RECORD_COLUMN = "tissue_record_column"; public static Map getColumnNameMap() { - DBElement dbElement = new DBElement("ddp_medical_record", "m", "pr", MEDICAL_RECORD_COLUMN); + DBElement medicalRecord = new DBElement("ddp_medical_record", "m", "pr", MEDICAL_RECORD_COLUMN); + DBElement tissueRecord = new DBElement("ddp_tissue_record", "t", "pr", TISSUE_RECORD_COLUMN); return Map.of( - MEDICAL_RECORD_COLUMN, dbElement + MEDICAL_RECORD_COLUMN, medicalRecord, + TISSUE_RECORD_COLUMN, tissueRecord ); } From efd8622d7f53e1d1fb45821a2d82700de8d1f5b4 Mon Sep 17 00:00:00 2001 From: Nika Date: Fri, 29 Oct 2021 10:45:38 +0400 Subject: [PATCH 070/405] DDP-7047 | make CollectionProcessor extend SourceGenerator in order to use collect() --- .../dsm/model/elastic/ESDsm.java | 3 +++ .../elastic/export/CollectionProcessor.java | 19 ++++++++++------- .../model/elastic/export/ExportFacade.java | 5 +++-- .../dsm/model/patch/ExistingRecordPatch.java | 2 +- .../dsm/model/patch/ParticipantDataPatch.java | 2 ++ .../export/CollectionProcessorTest.java | 21 ++++++++++++++++--- 6 files changed, 39 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java index ffde4b11f..602c75449 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java @@ -43,4 +43,7 @@ public class ESDsm { @SerializedName(ESObjectConstants.ONC_HISTORY_DETAIL_RECORDS) List> oncHistoryDetailRecords; + @SerializedName(ESObjectConstants.PARTICIPANT_DATA) + List> participantData; + } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java index 377894633..530853047 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java @@ -6,7 +6,7 @@ import java.util.*; import java.util.function.Predicate; -public class CollectionProcessor implements Processor { +public class CollectionProcessor extends SourceGenerator implements Processor { private ESDsm esDsm; private String propertyName; @@ -14,7 +14,8 @@ public class CollectionProcessor implements Processor { private final Predicate isFieldMatchProperty = field -> propertyName.equals(field.getName()); - public CollectionProcessor(ESDsm esDsm, String propertyName, GeneratorPayload generatorPayload) { + public CollectionProcessor(ESDsm esDsm, String propertyName, GeneratorPayload generatorPayload, Parser parser) { + super (parser, generatorPayload); this.esDsm = Objects.requireNonNull(esDsm); this.propertyName = Objects.requireNonNull(propertyName); this.generatorPayload = Objects.requireNonNull(generatorPayload); @@ -23,7 +24,6 @@ public CollectionProcessor(ESDsm esDsm, String propertyName, GeneratorPayload ge @Override public List> process() { List> fetchedRecords = extractDataByReflection(); - Map.of(propertyName, fetchedRecords); return updateIfExistsOrPut(fetchedRecords); } @@ -56,10 +56,15 @@ private List> updateIfExistsOrPut(List> } private void addNewRecordTo(List> fetchedRecords) { - fetchedRecords.add(Map.of( - MappingGenerator.ID, generatorPayload.getRecordId(), - generatorPayload.getNameValue().getName(), generatorPayload.getNameValue().getValue() - )); + Object collectedData = collect(); + if (collectedData instanceof Map) { + Map recordMap = (Map) collectedData; + recordMap.put(MappingGenerator.ID, generatorPayload.getRecordId()); + fetchedRecords.add(recordMap); + } else { + List> recordList = (List>) collectedData; + fetchedRecords.addAll(recordList); + } } private boolean isExistingRecord(Map eachRecord) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 09b33b966..294482005 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -46,10 +46,11 @@ private ESDsm fetchData() { private Map processData(ESDsm esDsm) { BaseGenerator.PropertyInfo propertyInfo = getPropertyInfo(); - generator = new SourceGenerator(new ValueParser(), exportFacadePayload.getGeneratorPayload()); + ValueParser valueParser = new ValueParser(); + generator = new SourceGenerator(valueParser, exportFacadePayload.getGeneratorPayload()); Map dataToReturn = generator.generate(); if (propertyInfo.isCollection()) { - processor = new CollectionProcessor(esDsm, propertyInfo.getPropertyName(), exportFacadePayload.getGeneratorPayload()); + processor = new CollectionProcessor(esDsm, propertyInfo.getPropertyName(), exportFacadePayload.getGeneratorPayload(), valueParser); List> processedData = processor.process(); if (!processedData.isEmpty()) { dataToReturn = Map.of(MappingGenerator.DSM_OBJECT, Map.of(propertyInfo.getPropertyName(), processedData)); diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java index b4618c82c..33ad22e10 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java @@ -179,7 +179,7 @@ Object handleSingleNameValue() { List nameValues = new ArrayList<>(); if (Patch.patch(patch.getId(), patch.getUser(), patch.getNameValue(), dbElement)) { nameValues.addAll(setWorkflowRelatedFields(patch)); -// writeDSMRecordsToES(); + writeDSMRecordsToES(); exportToES(patch.getNameValue()); //return nameValues with nulls return nameValues; diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantDataPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantDataPatch.java index 325195c54..f17b283ab 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantDataPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantDataPatch.java @@ -47,6 +47,8 @@ Optional processEachNameValue(NameValue nameValue) { } else if (participantDataId != null) { Patch.patch(participantDataId, patch.getUser(), nameValue, dbElement); + patch.setId(participantDataId); + exportToES(nameValue); } if (patch.getActions() != null) { profile = ElasticSearchUtil.getParticipantProfileByGuidOrAltPid(ddpInstance.getParticipantIndexES(), patch.getParentId()) diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessorTest.java index 1642426a8..cacd8d8ee 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessorTest.java @@ -1,6 +1,7 @@ package org.broadinstitute.dsm.model.elastic.export; import com.fasterxml.jackson.databind.ObjectMapper; +import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.elastic.ESDsm; import org.junit.Assert; @@ -26,7 +27,8 @@ public void testProcess() throws IOException { NameValue nameValue = new NameValue("mr", "mr_updated"); GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, (int)recordId); - CollectionProcessor collectionProcessor = new CollectionProcessor(esDsm, propertyName, generatorPayload); + ValueParser valueParser = new ValueParser(); + CollectionProcessor collectionProcessor = new CollectionProcessor(esDsm, propertyName, generatorPayload, valueParser); List> updatedList = collectionProcessor.process(); @@ -47,15 +49,28 @@ public void updateIfExistsOrPut() throws IOException { ESDsm esDsm = objectMapper.readValue(json, ESDsm.class); - NameValue nameValue = new NameValue("mr", "mr_updated"); + NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "val"); GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, 10); - CollectionProcessor collectionProcessor = new CollectionProcessor(esDsm, propertyName, generatorPayload); + CollectionProcessor collectionProcessor = new TestCollectionProcessor(esDsm, propertyName, new ValueParser(), generatorPayload); List> updatedList = collectionProcessor.process(); Assert.assertEquals(2, updatedList.size()); } + + private static class TestCollectionProcessor extends CollectionProcessor { + + public TestCollectionProcessor(ESDsm esDsm,String propertyName, Parser parser, GeneratorPayload generatorPayload) { + super(esDsm, propertyName, generatorPayload, parser); + } + + @Override + protected DBElement getDBElement() { + return TestPatchUtil.getColumnNameMap().get(getNameValue().getName()); + } + } + } From 9e57124261f5321cf346754dcf049e7a20ce8d4e Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 29 Oct 2021 15:06:27 +0400 Subject: [PATCH 071/405] DDP-7047 | change functionality to update several records, add tests to cover more cases, do minor refactoring --- .../elastic/export/CollectionProcessor.java | 21 +++++++++++--- .../model/elastic/export/SourceGenerator.java | 4 +-- .../dsm/model/patch/BasePatch.java | 7 ++++- .../dsm/model/patch/ExistingRecordPatch.java | 4 +-- .../model/patch/OncHistoryDetailPatch.java | 6 ++-- .../dsm/model/patch/ParticipantDataPatch.java | 3 +- .../model/patch/ParticipantRecordPatch.java | 1 + .../dsm/model/patch/TissuePatch.java | 3 +- .../export/CollectionProcessorTest.java | 29 +++++++++++++++++++ 9 files changed, 60 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java index 530853047..293159c11 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java @@ -5,6 +5,7 @@ import java.lang.reflect.Field; import java.util.*; import java.util.function.Predicate; +import java.util.stream.Collectors; public class CollectionProcessor extends SourceGenerator implements Processor { @@ -48,10 +49,14 @@ private List> getRecordsByField(Field field) { } private List> updateIfExistsOrPut(List> fetchedRecords) { - fetchedRecords.stream() + List> existingRecords = fetchedRecords.stream() .filter(this::isExistingRecord) - .findFirst() - .ifPresentOrElse(this::updateExistingRecord, () -> addNewRecordTo(fetchedRecords)); + .collect(Collectors.toList()); + if (existingRecords.isEmpty()) { + addNewRecordTo(fetchedRecords); + } else { + existingRecords.forEach(this::updateExistingRecord); + } return fetchedRecords; } @@ -72,6 +77,14 @@ private boolean isExistingRecord(Map eachRecord) { } private void updateExistingRecord(Map eachRecord) { - eachRecord.put(generatorPayload.getNameValue().getName(), generatorPayload.getNameValue().getValue()); + Object collectedData = collect(); + if (collectedData instanceof Map) { + Map recordMap = (Map) collectedData; + recordMap.put(MappingGenerator.ID, generatorPayload.getRecordId()); + eachRecord.putAll(recordMap); + } else { + List> records = (List>) collectedData; + records.forEach(eachRecord::putAll); + } } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java index 5db0f643a..efd4ed0ea 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java @@ -51,9 +51,7 @@ protected Object getElementWithId(Object element) { @Override protected Map getElement(Object element) { - return Map.of( - getOuterPropertyByAlias().getPropertyName(), Map.of(getDBElement().getColumnName(), - element)); + return Map.of(getDBElement().getColumnName(), element); } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index ee8bfc10b..8ca35de22 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -68,7 +68,7 @@ protected BasePatch(Patch patch) { prepareCommonData(); } - protected void exportToES(NameValue nameValue) { + private void exportToES(NameValue nameValue) { GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, Integer.parseInt(patch.getId())); ExportFacadePayload exportFacadePayload = new ExportFacadePayload(ddpInstance.getParticipantIndexES(), patch.getDdpParticipantId(), generatorPayload); @@ -116,6 +116,11 @@ List processMultipleNameValues() { return updatedNameValues; } + protected void exportToESWithId(String id) { + patch.setId(id); + exportToES(patch.getNameValue()); + } + abstract Optional processEachNameValue(NameValue nameValue); protected boolean hasQuestion(NameValue nameValue) { diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java index 33ad22e10..33159d7ac 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java @@ -60,7 +60,7 @@ public Object patchNameValuePair() { Optional processEachNameValue(NameValue nameValue) { Optional maybeUpdatedNameValue = Optional.empty(); Patch.patch(patch.getId(), patch.getUser(), nameValue, dbElement); - exportToES(nameValue); + exportToESWithId(patch.getId()); if (hasQuestion(nameValue)) { maybeUpdatedNameValue = sendNotificationEmailAndUpdateStatus(patch, nameValue, dbElement); } @@ -180,7 +180,7 @@ Object handleSingleNameValue() { if (Patch.patch(patch.getId(), patch.getUser(), patch.getNameValue(), dbElement)) { nameValues.addAll(setWorkflowRelatedFields(patch)); writeDSMRecordsToES(); - exportToES(patch.getNameValue()); + exportToESWithId(patch.getId()); //return nameValues with nulls return nameValues; } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/OncHistoryDetailPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/OncHistoryDetailPatch.java index 6fac50cfa..04a6d637a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/OncHistoryDetailPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/OncHistoryDetailPatch.java @@ -76,8 +76,7 @@ protected Object patchNameValuePair() { Object handleSingleNameValue() { if (Patch.patch(oncHistoryDetailId, patch.getUser(), patch.getNameValue(), dbElement)) { nameValues.addAll(setWorkflowRelatedFields(patch)); - patch.setId(mrID.toString()); - exportToES(patch.getNameValue()); + exportToESWithId(mrID.toString()); //set oncHistoryDetails created if it is a oncHistoryDetails value without a ID, otherwise created should already be set if (dbElement.getTableName().equals(DBConstants.DDP_ONC_HISTORY_DETAIL)) { NameValue oncHistoryCreated = OncHistory.setOncHistoryCreated(patch.getParentId(), patch.getUser()); @@ -93,8 +92,7 @@ Object handleSingleNameValue() { @Override Optional processEachNameValue(NameValue nameValue) { Patch.patch(oncHistoryDetailId, patch.getUser(), nameValue, dbElement); - patch.setId(mrID.toString()); - exportToES(nameValue); + exportToESWithId(mrID.toString()); return Optional.empty(); } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantDataPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantDataPatch.java index f17b283ab..1bd781bbe 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantDataPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantDataPatch.java @@ -47,8 +47,7 @@ Optional processEachNameValue(NameValue nameValue) { } else if (participantDataId != null) { Patch.patch(participantDataId, patch.getUser(), nameValue, dbElement); - patch.setId(participantDataId); - exportToES(nameValue); + exportToESWithId(participantDataId); } if (patch.getActions() != null) { profile = ElasticSearchUtil.getParticipantProfileByGuidOrAltPid(ddpInstance.getParticipantIndexES(), patch.getParentId()) diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantRecordPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantRecordPatch.java index a50a5e50b..b676b0ced 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantRecordPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantRecordPatch.java @@ -65,6 +65,7 @@ private void insertDdpParticipantRecord(int participantId) { @Override Object handleSingleNameValue() { Patch.patch(String.valueOf(participantId), patch.getUser(), patch.getNameValue(), dbElement); + exportToESWithId(String.valueOf(participantId)); resultMap.put(PARTICIPANT_ID, String.valueOf(participantId)); return resultMap; } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java index 82998fbb5..97c35457f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java @@ -44,8 +44,7 @@ Object handleSingleNameValue() { if (!nameValues.isEmpty()) { resultMap.put(NAME_VALUE, GSON.toJson(nameValues)); } - patch.setId(tissueId); - exportToES(patch.getNameValue()); + exportToESWithId(tissueId); } return resultMap; } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessorTest.java index cacd8d8ee..1ccd6d878 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessorTest.java @@ -10,6 +10,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.Optional; public class CollectionProcessorTest { @@ -60,6 +61,34 @@ public void updateIfExistsOrPut() throws IOException { } + @Test + public void updateIfExists() throws IOException { + String propertyName = "medicalRecords"; + double recordId = 5; + String json = String.format("{\"%s\":[{\"id\":%s,\"TEST1\":\"%s\", \"TEST2\":\"TEST_VAL2\"}]}", propertyName, recordId, "value");; + + ObjectMapper objectMapper = new ObjectMapper(); + + ESDsm esDsm = objectMapper.readValue(json, ESDsm.class); + + NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "{\"TEST1\":\"TEST_VAL\", \"TEST2\":\"TEST_VAL3\"}"); + GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, 5); + + CollectionProcessor collectionProcessor = new TestCollectionProcessor(esDsm, propertyName, new ValueParser(), generatorPayload); + + List> updatedList = collectionProcessor.process(); + + updatedList.stream() + .filter(i -> i.containsKey("TEST1")) + .findFirst() + .ifPresentOrElse(m -> m.get("TEST1").equals("TEST_VAL"), Assert::fail); + + updatedList.stream() + .filter(i -> i.containsKey("TEST2")) + .findFirst() + .ifPresentOrElse(m -> m.get("TEST2").equals("TEST_VAL3"), Assert::fail); + } + private static class TestCollectionProcessor extends CollectionProcessor { public TestCollectionProcessor(ESDsm esDsm,String propertyName, Parser parser, GeneratorPayload generatorPayload) { From 66bcb84cb5573791e634d3b28621878446241d42 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 29 Oct 2021 16:22:36 +0400 Subject: [PATCH 072/405] DDP-7047 | add fields to ESDsm, eliminate dead code --- .../dsm/model/elastic/ESDsm.java | 10 +++++ .../dsm/model/patch/ExistingRecordPatch.java | 40 ------------------- 2 files changed, 10 insertions(+), 40 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java index 602c75449..c0ddfab09 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java @@ -43,7 +43,17 @@ public class ESDsm { @SerializedName(ESObjectConstants.ONC_HISTORY_DETAIL_RECORDS) List> oncHistoryDetailRecords; + @SerializedName(ESObjectConstants.ONC_HISTORY) + Map oncHistory; + @SerializedName(ESObjectConstants.PARTICIPANT_DATA) List> participantData; + @SerializedName(ESObjectConstants.PARTICIPANT_RECORD) + Map participantRecord; + + @SerializedName(ESObjectConstants.PARTICIPANT) + Map participant; + + } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java index 33159d7ac..3503a654f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java @@ -179,49 +179,9 @@ Object handleSingleNameValue() { List nameValues = new ArrayList<>(); if (Patch.patch(patch.getId(), patch.getUser(), patch.getNameValue(), dbElement)) { nameValues.addAll(setWorkflowRelatedFields(patch)); - writeDSMRecordsToES(); exportToESWithId(patch.getId()); - //return nameValues with nulls return nameValues; } return nameValues; } - - private void writeDSMRecordsToES() { - NameValue nameValue = patch.getNameValue(); - String name = nameValue.getName().substring(nameValue.getName().lastIndexOf('.') + 1); - String type = getTypeFrom(nameValue); - if (type == null) return; - String value = nameValue.getValue().toString(); - Map nameValueMap = new HashMap<>(); - nameValueMap.put(name, value); - if (isMedicalRecord(name, type)) { - ElasticSearchUtil.writeDsmRecord(ddpInstance, Integer.parseInt(patch.getId()), patch.getDdpParticipantId(), - ESObjectConstants.MEDICAL_RECORDS, ESObjectConstants.MEDICAL_RECORDS_ID, nameValueMap); - } - else if (DBConstants.DDP_ONC_HISTORY_DETAIL_ALIAS.equals(type)) { - // TODO relationship between DDP_ONC_HISTORY_DETAIL_ALIAS and TISSUE_RECORDS_FIELD_NAMES - if (ESObjectConstants.TISSUE_RECORDS_FIELD_NAMES.contains(name)) { - if (PARTICIPANT_ID.equals(patch.getParent())) { - ElasticSearchUtil.writeDsmRecord(ddpInstance, Integer.parseInt(patch.getId()), patch.getDdpParticipantId(), - ESObjectConstants.TISSUE_RECORDS, ESObjectConstants.TISSUE_RECORDS_ID, nameValueMap); - } - } - } - } - - private String getTypeFrom(NameValue nameValue) { - String type = null; - if (nameValue.getName().indexOf('.') != -1) { - type = nameValue.getName().substring(0, nameValue.getName().indexOf('.')); - } - else { - return null; - } - return type; - } - - private boolean isMedicalRecord(String name, String type) { - return DBConstants.DDP_MEDICAL_RECORD_ALIAS.equals(type) && ESObjectConstants.MEDICAL_RECORDS_FIELD_NAMES.contains(name); - } } From 4019f190af29a5ac979a54dfe3a516ccd9835bf8 Mon Sep 17 00:00:00 2001 From: Nika Date: Fri, 29 Oct 2021 18:05:46 +0400 Subject: [PATCH 073/405] DDP-7047 | pass NameValues appropriately, where needed --- .../dsm/model/elastic/export/ExportFacade.java | 2 ++ .../model/elastic/export/ExportFacadePayload.java | 4 ++++ .../dsm/model/elastic/search/ElasticSearch.java | 10 +++++----- .../broadinstitute/dsm/model/patch/BasePatch.java | 12 ++++-------- .../dsm/model/patch/ExistingRecordPatch.java | 4 ++-- .../dsm/model/patch/OncHistoryDetailPatch.java | 4 ++-- .../dsm/model/patch/ParticipantDataPatch.java | 2 +- .../dsm/model/patch/ParticipantRecordPatch.java | 2 +- .../broadinstitute/dsm/model/patch/TissuePatch.java | 2 +- 9 files changed, 22 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 294482005..82ccf99bc 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -41,6 +41,8 @@ private void upsertMapping() { private ESDsm fetchData() { ElasticSearchParticipantDto participantById = searchable.getParticipantById(exportFacadePayload.getIndex(), exportFacadePayload.getDocId()); + // Ensure that participant data will be stored by participant guid + exportFacadePayload.setDocId(participantById.getParticipantId()); return participantById.getDsm().orElseThrow(); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java index 129cbb225..9b22d0b3e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java @@ -26,4 +26,8 @@ public String getDocId() { public GeneratorPayload getGeneratorPayload() { return generatorPayload; } + + public void setDocId(String docId) { + this.docId = docId; + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java index 37abcee0c..4f10c7410 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java @@ -194,11 +194,11 @@ public ElasticSearch getParticipantsByRangeAndIds(String participantIndexES, int } @Override - public ElasticSearchParticipantDto getParticipantById(String esParticipantsIndex, String shortId) { - String type = Util.getQueryTypeFromId(shortId); - String id = Objects.requireNonNull(shortId); + public ElasticSearchParticipantDto getParticipantById(String esParticipantsIndex, String id) { + String type = Util.getQueryTypeFromId(id); + String participantId = Objects.requireNonNull(id); SearchRequest searchRequest = new SearchRequest(Objects.requireNonNull(esParticipantsIndex)); - TermQueryBuilder shortIdQuery = QueryBuilders.termQuery(type, id); + TermQueryBuilder shortIdQuery = QueryBuilders.termQuery(type, participantId); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); searchSourceBuilder.query(shortIdQuery); searchRequest.source(searchSourceBuilder); @@ -209,7 +209,7 @@ public ElasticSearchParticipantDto getParticipantById(String esParticipantsIndex searchResponse = ElasticSearchUtil.getClientInstance().search(searchRequest, RequestOptions.DEFAULT); sourceAsMap = searchResponse.getHits().getHits()[0].getSourceAsMap(); } catch (Exception e) { - throw new RuntimeException("Couldn't get participant from ES for instance " + esParticipantsIndex + " by short id: " + shortId, e); + throw new RuntimeException("Couldn't get participant from ES for instance " + esParticipantsIndex + " by short id: " + participantId, e); } return parseSourceMap(sourceAsMap).orElseThrow(); } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index 8ca35de22..07226746e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -2,11 +2,7 @@ import static org.broadinstitute.ddp.db.TransactionWrapper.inTransaction; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; +import java.util.*; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -116,9 +112,9 @@ List processMultipleNameValues() { return updatedNameValues; } - protected void exportToESWithId(String id) { - patch.setId(id); - exportToES(patch.getNameValue()); + protected void exportToESWithId(String id, NameValue nameValue) { + patch.setId(Objects.requireNonNull(id)); + exportToES(Objects.requireNonNull(nameValue)); } abstract Optional processEachNameValue(NameValue nameValue); diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java index 3503a654f..0ede74fa2 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java @@ -60,7 +60,7 @@ public Object patchNameValuePair() { Optional processEachNameValue(NameValue nameValue) { Optional maybeUpdatedNameValue = Optional.empty(); Patch.patch(patch.getId(), patch.getUser(), nameValue, dbElement); - exportToESWithId(patch.getId()); + exportToESWithId(patch.getId(), nameValue); if (hasQuestion(nameValue)) { maybeUpdatedNameValue = sendNotificationEmailAndUpdateStatus(patch, nameValue, dbElement); } @@ -179,7 +179,7 @@ Object handleSingleNameValue() { List nameValues = new ArrayList<>(); if (Patch.patch(patch.getId(), patch.getUser(), patch.getNameValue(), dbElement)) { nameValues.addAll(setWorkflowRelatedFields(patch)); - exportToESWithId(patch.getId()); + exportToESWithId(patch.getId(), patch.getNameValue()); return nameValues; } return nameValues; diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/OncHistoryDetailPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/OncHistoryDetailPatch.java index 04a6d637a..63e26aad6 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/OncHistoryDetailPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/OncHistoryDetailPatch.java @@ -76,7 +76,7 @@ protected Object patchNameValuePair() { Object handleSingleNameValue() { if (Patch.patch(oncHistoryDetailId, patch.getUser(), patch.getNameValue(), dbElement)) { nameValues.addAll(setWorkflowRelatedFields(patch)); - exportToESWithId(mrID.toString()); + exportToESWithId(mrID.toString(), patch.getNameValue()); //set oncHistoryDetails created if it is a oncHistoryDetails value without a ID, otherwise created should already be set if (dbElement.getTableName().equals(DBConstants.DDP_ONC_HISTORY_DETAIL)) { NameValue oncHistoryCreated = OncHistory.setOncHistoryCreated(patch.getParentId(), patch.getUser()); @@ -92,7 +92,7 @@ Object handleSingleNameValue() { @Override Optional processEachNameValue(NameValue nameValue) { Patch.patch(oncHistoryDetailId, patch.getUser(), nameValue, dbElement); - exportToESWithId(mrID.toString()); + exportToESWithId(mrID.toString(), nameValue); return Optional.empty(); } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantDataPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantDataPatch.java index 1bd781bbe..3ceb6055d 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantDataPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantDataPatch.java @@ -47,7 +47,7 @@ Optional processEachNameValue(NameValue nameValue) { } else if (participantDataId != null) { Patch.patch(participantDataId, patch.getUser(), nameValue, dbElement); - exportToESWithId(participantDataId); + exportToESWithId(participantDataId, nameValue); } if (patch.getActions() != null) { profile = ElasticSearchUtil.getParticipantProfileByGuidOrAltPid(ddpInstance.getParticipantIndexES(), patch.getParentId()) diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantRecordPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantRecordPatch.java index b676b0ced..f1af239bf 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantRecordPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantRecordPatch.java @@ -65,7 +65,7 @@ private void insertDdpParticipantRecord(int participantId) { @Override Object handleSingleNameValue() { Patch.patch(String.valueOf(participantId), patch.getUser(), patch.getNameValue(), dbElement); - exportToESWithId(String.valueOf(participantId)); + exportToESWithId(String.valueOf(participantId), patch.getNameValue()); resultMap.put(PARTICIPANT_ID, String.valueOf(participantId)); return resultMap; } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java index 97c35457f..bfc5ecf71 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/TissuePatch.java @@ -44,7 +44,7 @@ Object handleSingleNameValue() { if (!nameValues.isEmpty()) { resultMap.put(NAME_VALUE, GSON.toJson(nameValues)); } - exportToESWithId(tissueId); + exportToESWithId(tissueId, patch.getNameValue()); } return resultMap; } From 150f0073300f760376ac404b1080054884d40950 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 29 Oct 2021 18:40:18 +0400 Subject: [PATCH 074/405] DDP-7047 | reorganize files into packages --- .../org/broadinstitute/dsm/model/elastic/Util.java | 4 +--- .../dsm/model/elastic/export/ExportFacade.java | 8 ++++++++ .../model/elastic/export/ExportFacadePayload.java | 2 +- .../dsm/model/elastic/export/Generator.java | 9 --------- .../export/{ => generate}/BaseGenerator.java | 3 ++- .../model/elastic/export/generate/Generator.java | 7 +++++++ .../export/{ => generate}/GeneratorPayload.java | 2 +- .../export/{ => generate}/MappingGenerator.java | 4 ++-- .../export/{ => generate}/SourceGenerator.java | 4 ++-- .../elastic/export/{ => parse}/BaseParser.java | 2 +- .../model/elastic/export/{ => parse}/Parser.java | 2 +- .../elastic/export/{ => parse}/TypeParser.java | 2 +- .../elastic/export/{ => parse}/ValueParser.java | 2 +- .../export/{ => process}/CollectionProcessor.java | 7 ++++++- .../elastic/export/{ => process}/Processor.java | 2 +- .../broadinstitute/dsm/model/patch/BasePatch.java | 1 + .../dsm/model/patch/PatchFactory.java | 4 ---- .../dsm/model/elastic/export/TestPatchUtil.java | 2 +- .../{ => generate}/MappingGeneratorTest.java | 5 ++++- .../export/{ => generate}/SourceGeneratorTest.java | 5 ++++- .../{ => process}/CollectionProcessorTest.java | 14 +++++++++----- 21 files changed, 54 insertions(+), 37 deletions(-) delete mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/Generator.java rename src/main/java/org/broadinstitute/dsm/model/elastic/export/{ => generate}/BaseGenerator.java (95%) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Generator.java rename src/main/java/org/broadinstitute/dsm/model/elastic/export/{ => generate}/GeneratorPayload.java (86%) rename src/main/java/org/broadinstitute/dsm/model/elastic/export/{ => generate}/MappingGenerator.java (94%) rename src/main/java/org/broadinstitute/dsm/model/elastic/export/{ => generate}/SourceGenerator.java (93%) rename src/main/java/org/broadinstitute/dsm/model/elastic/export/{ => parse}/BaseParser.java (96%) rename src/main/java/org/broadinstitute/dsm/model/elastic/export/{ => parse}/Parser.java (51%) rename src/main/java/org/broadinstitute/dsm/model/elastic/export/{ => parse}/TypeParser.java (87%) rename src/main/java/org/broadinstitute/dsm/model/elastic/export/{ => parse}/ValueParser.java (88%) rename src/main/java/org/broadinstitute/dsm/model/elastic/export/{ => process}/CollectionProcessor.java (89%) rename src/main/java/org/broadinstitute/dsm/model/elastic/export/{ => process}/Processor.java (66%) rename src/test/java/org/broadinstitute/dsm/model/elastic/export/{ => generate}/MappingGeneratorTest.java (95%) rename src/test/java/org/broadinstitute/dsm/model/elastic/export/{ => generate}/SourceGeneratorTest.java (92%) rename src/test/java/org/broadinstitute/dsm/model/elastic/export/{ => process}/CollectionProcessorTest.java (82%) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 96406ad99..983fe8a67 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -3,11 +3,9 @@ import java.util.Map; import java.util.Objects; -import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.structure.DBElement; -import org.broadinstitute.dsm.model.elastic.export.BaseGenerator; +import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; import org.broadinstitute.dsm.statics.ESObjectConstants; -import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.broadinstitute.dsm.util.ParticipantUtil; import org.broadinstitute.dsm.util.PatchUtil; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 82ccf99bc..27857e0a2 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -7,6 +7,14 @@ import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.elastic.ESDsm; import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; +import org.broadinstitute.dsm.model.elastic.export.generate.Generator; +import org.broadinstitute.dsm.model.elastic.export.generate.MappingGenerator; +import org.broadinstitute.dsm.model.elastic.export.generate.SourceGenerator; +import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; +import org.broadinstitute.dsm.model.elastic.export.parse.ValueParser; +import org.broadinstitute.dsm.model.elastic.export.process.CollectionProcessor; +import org.broadinstitute.dsm.model.elastic.export.process.Processor; import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchable; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java index 9b22d0b3e..55f50ef78 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java @@ -2,7 +2,7 @@ import java.util.Objects; -import org.broadinstitute.dsm.model.NameValue; +import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; public class ExportFacadePayload { private String index; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Generator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/Generator.java deleted file mode 100644 index 01a478a4a..000000000 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Generator.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.broadinstitute.dsm.model.elastic.export; - -import java.util.Map; - -import org.broadinstitute.dsm.model.NameValue; - -public interface Generator { - Map generate(); -} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java similarity index 95% rename from src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java index aaa469d3a..518981bad 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java @@ -1,10 +1,11 @@ -package org.broadinstitute.dsm.model.elastic.export; +package org.broadinstitute.dsm.model.elastic.export.generate; import com.google.gson.Gson; import com.google.gson.JsonSyntaxException; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import java.util.Map; import java.util.Objects; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Generator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Generator.java new file mode 100644 index 000000000..01cc63886 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Generator.java @@ -0,0 +1,7 @@ +package org.broadinstitute.dsm.model.elastic.export.generate; + +import java.util.Map; + +public interface Generator { + Map generate(); +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/GeneratorPayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorPayload.java similarity index 86% rename from src/main/java/org/broadinstitute/dsm/model/elastic/export/GeneratorPayload.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorPayload.java index 81438c71e..75ab7cdec 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/GeneratorPayload.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorPayload.java @@ -1,4 +1,4 @@ -package org.broadinstitute.dsm.model.elastic.export; +package org.broadinstitute.dsm.model.elastic.export.generate; import org.broadinstitute.dsm.model.NameValue; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java similarity index 94% rename from src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java index 99c912535..949ec5e0e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java @@ -1,9 +1,9 @@ -package org.broadinstitute.dsm.model.elastic.export; +package org.broadinstitute.dsm.model.elastic.export.generate; import java.util.HashMap; import java.util.Map; -import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; public class MappingGenerator extends BaseGenerator { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java similarity index 93% rename from src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java index efd4ed0ea..0643d0cb0 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java @@ -1,10 +1,10 @@ -package org.broadinstitute.dsm.model.elastic.export; +package org.broadinstitute.dsm.model.elastic.export.generate; import java.util.HashMap; import java.util.List; import java.util.Map; -import org.jcodings.util.Hash; +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; public class SourceGenerator extends BaseGenerator { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java similarity index 96% rename from src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseParser.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java index bd32f6e21..6169c0e39 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java @@ -1,4 +1,4 @@ -package org.broadinstitute.dsm.model.elastic.export; +package org.broadinstitute.dsm.model.elastic.export.parse; import java.time.LocalDate; import java.time.LocalDateTime; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Parser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/Parser.java similarity index 51% rename from src/main/java/org/broadinstitute/dsm/model/elastic/export/Parser.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/Parser.java index 0853ee018..66d3d1911 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Parser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/Parser.java @@ -1,4 +1,4 @@ -package org.broadinstitute.dsm.model.elastic.export; +package org.broadinstitute.dsm.model.elastic.export.parse; public interface Parser { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/TypeParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java similarity index 87% rename from src/main/java/org/broadinstitute/dsm/model/elastic/export/TypeParser.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java index 231ea66e7..90ecfcc8b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/TypeParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java @@ -1,4 +1,4 @@ -package org.broadinstitute.dsm.model.elastic.export; +package org.broadinstitute.dsm.model.elastic.export.parse; public class TypeParser extends BaseParser { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ValueParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/ValueParser.java similarity index 88% rename from src/main/java/org/broadinstitute/dsm/model/elastic/export/ValueParser.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/ValueParser.java index 038f9cd3b..10fd5ceb3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ValueParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/ValueParser.java @@ -1,4 +1,4 @@ -package org.broadinstitute.dsm.model.elastic.export; +package org.broadinstitute.dsm.model.elastic.export.parse; public class ValueParser extends BaseParser { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java similarity index 89% rename from src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java index 293159c11..caa1c1d01 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java @@ -1,6 +1,11 @@ -package org.broadinstitute.dsm.model.elastic.export; +package org.broadinstitute.dsm.model.elastic.export.process; import org.broadinstitute.dsm.model.elastic.ESDsm; +import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; +import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; +import org.broadinstitute.dsm.model.elastic.export.generate.MappingGenerator; +import org.broadinstitute.dsm.model.elastic.export.generate.SourceGenerator; +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import java.lang.reflect.Field; import java.util.*; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Processor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/Processor.java similarity index 66% rename from src/main/java/org/broadinstitute/dsm/model/elastic/export/Processor.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/export/process/Processor.java index fdd97ea1b..80edc8db6 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Processor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/Processor.java @@ -1,4 +1,4 @@ -package org.broadinstitute.dsm.model.elastic.export; +package org.broadinstitute.dsm.model.elastic.export.process; import java.util.List; import java.util.Map; diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index 07226746e..979f626a4 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -21,6 +21,7 @@ import org.broadinstitute.dsm.model.Value; import org.broadinstitute.dsm.model.elastic.ESProfile; import org.broadinstitute.dsm.model.elastic.export.*; +import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java index 69bca6081..bc34f14a8 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java @@ -1,10 +1,6 @@ package org.broadinstitute.dsm.model.patch; import org.apache.commons.lang3.StringUtils; -import org.broadinstitute.dsm.model.elastic.export.ElasticExportAdapter; -import org.broadinstitute.dsm.model.elastic.export.MappingGenerator; -import org.broadinstitute.dsm.model.elastic.export.SourceGenerator; -import org.broadinstitute.dsm.model.elastic.export.TypeParser; import org.broadinstitute.dsm.util.NotificationUtil; public class PatchFactory { diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/TestPatchUtil.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/TestPatchUtil.java index 7c17007a6..a5097575c 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/TestPatchUtil.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/TestPatchUtil.java @@ -5,7 +5,7 @@ import java.util.Map; -class TestPatchUtil extends PatchUtil { +public class TestPatchUtil extends PatchUtil { public static final String MEDICAL_RECORD_COLUMN = "medical_record_column"; public static final String TISSUE_RECORD_COLUMN = "tissue_record_column"; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java similarity index 95% rename from src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java rename to src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java index 2fea35cce..613705fb7 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/MappingGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java @@ -1,8 +1,11 @@ -package org.broadinstitute.dsm.model.elastic.export; +package org.broadinstitute.dsm.model.elastic.export.generate; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; +import org.broadinstitute.dsm.model.elastic.export.TestPatchUtil; +import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorTest.java similarity index 92% rename from src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java rename to src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorTest.java index 006e66117..fd3a1f536 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/SourceGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorTest.java @@ -1,4 +1,4 @@ -package org.broadinstitute.dsm.model.elastic.export; +package org.broadinstitute.dsm.model.elastic.export.generate; import java.util.List; import java.util.Map; @@ -6,6 +6,9 @@ import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; +import org.broadinstitute.dsm.model.elastic.export.TestPatchUtil; +import org.broadinstitute.dsm.model.elastic.export.parse.ValueParser; import org.junit.Test; import org.junit.Assert; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorTest.java similarity index 82% rename from src/test/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessorTest.java rename to src/test/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorTest.java index 1ccd6d878..916c0b332 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/CollectionProcessorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorTest.java @@ -1,16 +1,20 @@ -package org.broadinstitute.dsm.model.elastic.export; +package org.broadinstitute.dsm.model.elastic.export.process; import com.fasterxml.jackson.databind.ObjectMapper; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.elastic.ESDsm; +import org.broadinstitute.dsm.model.elastic.export.TestPatchUtil; +import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; +import org.broadinstitute.dsm.model.elastic.export.parse.ValueParser; +import org.broadinstitute.dsm.model.elastic.export.process.CollectionProcessor; import org.junit.Assert; import org.junit.Test; import java.io.IOException; import java.util.List; import java.util.Map; -import java.util.Optional; public class CollectionProcessorTest { @@ -25,11 +29,11 @@ public void testProcess() throws IOException { ESDsm esDsm = objectMapper.readValue(json, ESDsm.class); - NameValue nameValue = new NameValue("mr", "mr_updated"); + NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "mr_updated"); GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, (int)recordId); ValueParser valueParser = new ValueParser(); - CollectionProcessor collectionProcessor = new CollectionProcessor(esDsm, propertyName, generatorPayload, valueParser); + CollectionProcessor collectionProcessor = new TestCollectionProcessor(esDsm, propertyName, valueParser, generatorPayload); List> updatedList = collectionProcessor.process(); @@ -91,7 +95,7 @@ public void updateIfExists() throws IOException { private static class TestCollectionProcessor extends CollectionProcessor { - public TestCollectionProcessor(ESDsm esDsm,String propertyName, Parser parser, GeneratorPayload generatorPayload) { + public TestCollectionProcessor(ESDsm esDsm, String propertyName, Parser parser, GeneratorPayload generatorPayload) { super(esDsm, propertyName, generatorPayload, parser); } From ec90063abc7d1717aae841644886a8d37e0010de Mon Sep 17 00:00:00 2001 From: Nika Date: Fri, 29 Oct 2021 19:07:17 +0400 Subject: [PATCH 075/405] DDP-7047 | create template method for constructing final object data required by ES --- .../export/generate/BaseGenerator.java | 14 ++++++++++ .../export/generate/MappingGenerator.java | 22 +++++++-------- .../export/generate/SourceGenerator.java | 27 +++++++++++++------ 3 files changed, 44 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java index 518981bad..e4bd871ee 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java @@ -71,6 +71,20 @@ protected Object getFieldWithElement() { protected abstract Map getElement(Object element); + protected Object constructByPropertyType() { + Object constructedObject; + if(getOuterPropertyByAlias().isCollection()) { + constructedObject = constructCollection(); + } else { + constructedObject = constructSingleElement(); + } + return constructedObject; + } + + protected abstract Object constructSingleElement(); + + protected abstract Object constructCollection(); + public static class PropertyInfo { String propertyName; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java index 949ec5e0e..82bafae50 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java @@ -7,7 +7,6 @@ public class MappingGenerator extends BaseGenerator { - public static final String TYPE = "type"; public static final String NESTED = "nested"; public static final String TYPE_KEYWORD = "keyword"; @@ -28,16 +27,7 @@ public Map generate() { } private Map buildMappedField() { - PropertyInfo propertyInfo = getOuterPropertyByAlias(); - boolean isPropertyCollection = propertyInfo.isCollection(); - Object field = collect(); - Map mappedField; - if (isPropertyCollection) { - mappedField = Map.of(TYPE, NESTED, PROPERTIES, field); - } else { - mappedField = Map.of(PROPERTIES, field); - } - return mappedField; + return (Map) constructByPropertyType(); } @Override @@ -69,4 +59,14 @@ protected Map getElement(Object type) { return Map.of(getDBElement().getColumnName(), Map.of(MappingGenerator.TYPE, type)); } + @Override + protected Object constructSingleElement() { + return Map.of(PROPERTIES, collect()); + } + + @Override + protected Object constructCollection() { + return Map.of(TYPE, NESTED, PROPERTIES, collect()); + } + } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java index 0643d0cb0..d526a6fae 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java @@ -23,19 +23,16 @@ public Map generate() { @Override protected Object parseJson() { + return constructByPropertyType(); + } + + private Map parseJsonValuesToObject() { Map dynamicFieldValues = parseJsonToMapFromValue(); Map transformedMap = new HashMap<>(); for (Map.Entry entry : dynamicFieldValues.entrySet()) { transformedMap.put(entry.getKey(), parser.parse(String.valueOf(entry.getValue()))); } - if (getOuterPropertyByAlias().isCollection()) { - Map collectionMap = new HashMap<>(); - collectionMap.put(ID, generatorPayload.getRecordId()); - collectionMap.putAll(transformedMap); - return List.of(collectionMap); - } else { - return transformedMap; - } + return transformedMap; } @Override @@ -54,5 +51,19 @@ protected Map getElement(Object element) { return Map.of(getDBElement().getColumnName(), element); } + @Override + protected Object constructSingleElement() { + return parseJsonValuesToObject(); + } + + @Override + protected Object constructCollection() { + Map collectionMap = new HashMap<>(); + collectionMap.put(ID, generatorPayload.getRecordId()); + Map mapWithParsedObjects = parseJsonValuesToObject(); + collectionMap.putAll(mapWithParsedObjects); + return List.of(collectionMap); + } + } From 43cd4074a18ef06a3403d41341f56f6719cf8bfd Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 29 Oct 2021 19:42:11 +0400 Subject: [PATCH 076/405] DDP-7047 | introduce collector abstraction --- .../model/elastic/export/ExportFacade.java | 8 +++--- .../export/generate/BaseGenerator.java | 26 ++++++++++++++----- .../elastic/export/generate/Collector.java | 5 ++++ .../export/process/CollectionProcessor.java | 16 ++++++++---- .../export/generate/MappingGeneratorTest.java | 2 +- .../process/CollectionProcessorTest.java | 17 +++++++++--- 6 files changed, 54 insertions(+), 20 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Collector.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 27857e0a2..8caed38f8 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -57,10 +57,12 @@ private ESDsm fetchData() { private Map processData(ESDsm esDsm) { BaseGenerator.PropertyInfo propertyInfo = getPropertyInfo(); ValueParser valueParser = new ValueParser(); - generator = new SourceGenerator(valueParser, exportFacadePayload.getGeneratorPayload()); - Map dataToReturn = generator.generate(); + SourceGenerator sourceGenerator = new SourceGenerator(valueParser, exportFacadePayload.getGeneratorPayload()); + this.generator = sourceGenerator; + Map dataToReturn = this.generator.generate(); if (propertyInfo.isCollection()) { - processor = new CollectionProcessor(esDsm, propertyInfo.getPropertyName(), exportFacadePayload.getGeneratorPayload(), valueParser); + processor = new CollectionProcessor(esDsm, propertyInfo.getPropertyName(), exportFacadePayload.getGeneratorPayload(), + sourceGenerator); List> processedData = processor.process(); if (!processedData.isEmpty()) { dataToReturn = Map.of(MappingGenerator.DSM_OBJECT, Map.of(propertyInfo.getPropertyName(), processedData)); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java index e4bd871ee..a5266b6cc 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java @@ -10,7 +10,7 @@ import java.util.Map; import java.util.Objects; -public abstract class BaseGenerator implements Generator { +public abstract class BaseGenerator implements Generator, Collector { public static final String DSM_OBJECT = "dsm"; @@ -19,10 +19,17 @@ public abstract class BaseGenerator implements Generator { protected static final Gson GSON = new Gson(); protected final Parser parser; protected GeneratorPayload generatorPayload; + private DBElement dbElement; public BaseGenerator(Parser parser, GeneratorPayload generatorPayload) { this.parser = Objects.requireNonNull(parser); this.generatorPayload = Objects.requireNonNull(generatorPayload); + dbElement = Util.getDBElement(getNameValue().getName()); + } + + //setter method to set dbElement for testing + public void setDBElement(DBElement dbElement) { + this.dbElement = dbElement; } protected NameValue getNameValue() { @@ -31,14 +38,15 @@ protected NameValue getNameValue() { //wrap Util.getDBElement in protected method so that we can override it in testing class for tests protected DBElement getDBElement() { - return Util.getDBElement(getNameValue().getName()); + return dbElement; } protected PropertyInfo getOuterPropertyByAlias() { return Util.TABLE_ALIAS_MAPPINGS.get(getDBElement().getTableAlias()); } - - protected Object collect() { + + @Override + public Object collect() { Object sourceToUpsert; try { sourceToUpsert = parseJson(); @@ -59,7 +67,7 @@ protected Map parseJsonToMapFromValue() { protected Object getFieldWithElement() { Object fieldElementMap; Object element = parser.parse(String.valueOf(getNameValue().getValue())); - if (getOuterPropertyByAlias().isCollection) { + if (getOuterPropertyByAlias().isCollection()) { fieldElementMap = getElementWithId(element); } else { fieldElementMap = getElement(element); @@ -87,14 +95,18 @@ protected Object constructByPropertyType() { public static class PropertyInfo { - String propertyName; - boolean isCollection; + private String propertyName; + private boolean isCollection; public PropertyInfo(String propertyName, boolean isCollection) { this.propertyName = Objects.requireNonNull(propertyName); this.isCollection = isCollection; } + public void setIsCollection(boolean isCollection) { + this.isCollection = isCollection; + } + public String getPropertyName() { return propertyName; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Collector.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Collector.java new file mode 100644 index 000000000..4768a1cdf --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Collector.java @@ -0,0 +1,5 @@ +package org.broadinstitute.dsm.model.elastic.export.generate; + +public interface Collector { + Object collect(); +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java index caa1c1d01..ec9c70e81 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java @@ -1,6 +1,7 @@ package org.broadinstitute.dsm.model.elastic.export.process; import org.broadinstitute.dsm.model.elastic.ESDsm; +import org.broadinstitute.dsm.model.elastic.export.generate.Collector; import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; import org.broadinstitute.dsm.model.elastic.export.generate.MappingGenerator; @@ -12,19 +13,24 @@ import java.util.function.Predicate; import java.util.stream.Collectors; -public class CollectionProcessor extends SourceGenerator implements Processor { +public class CollectionProcessor implements Processor { private ESDsm esDsm; private String propertyName; private GeneratorPayload generatorPayload; + private Collector collector; private final Predicate isFieldMatchProperty = field -> propertyName.equals(field.getName()); - public CollectionProcessor(ESDsm esDsm, String propertyName, GeneratorPayload generatorPayload, Parser parser) { - super (parser, generatorPayload); + public CollectionProcessor(ESDsm esDsm, String propertyName, GeneratorPayload generatorPayload, Collector collector) { this.esDsm = Objects.requireNonNull(esDsm); this.propertyName = Objects.requireNonNull(propertyName); this.generatorPayload = Objects.requireNonNull(generatorPayload); + this.collector = collector; + } + + protected CollectionProcessor() { + } @Override @@ -66,7 +72,7 @@ private List> updateIfExistsOrPut(List> } private void addNewRecordTo(List> fetchedRecords) { - Object collectedData = collect(); + Object collectedData = collector.collect(); if (collectedData instanceof Map) { Map recordMap = (Map) collectedData; recordMap.put(MappingGenerator.ID, generatorPayload.getRecordId()); @@ -82,7 +88,7 @@ private boolean isExistingRecord(Map eachRecord) { } private void updateExistingRecord(Map eachRecord) { - Object collectedData = collect(); + Object collectedData = collector.collect(); if (collectedData instanceof Map) { Map recordMap = (Map) collectedData; recordMap.put(MappingGenerator.ID, generatorPayload.getRecordId()); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java index 613705fb7..771adbc65 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java @@ -79,7 +79,7 @@ public void getFieldWithTypeCollectionTrue() { 100 ); TestMappingGenerator generator = TestMappingGenerator.of(generatorPayload); - generator.getOuterPropertyByAlias().isCollection = true; + generator.getOuterPropertyByAlias().setIsCollection(true); Object fieldWithType = generator.getFieldWithElement(); Assert.assertTrue(((Map) fieldWithType).containsKey(BaseGenerator.ID)); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorTest.java index 916c0b332..25495e2b4 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorTest.java @@ -5,7 +5,9 @@ import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.elastic.ESDsm; import org.broadinstitute.dsm.model.elastic.export.TestPatchUtil; +import org.broadinstitute.dsm.model.elastic.export.generate.Collector; import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; +import org.broadinstitute.dsm.model.elastic.export.generate.SourceGenerator; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import org.broadinstitute.dsm.model.elastic.export.parse.ValueParser; import org.broadinstitute.dsm.model.elastic.export.process.CollectionProcessor; @@ -96,13 +98,20 @@ public void updateIfExists() throws IOException { private static class TestCollectionProcessor extends CollectionProcessor { public TestCollectionProcessor(ESDsm esDsm, String propertyName, Parser parser, GeneratorPayload generatorPayload) { - super(esDsm, propertyName, generatorPayload, parser); + SourceGenerator sourceGenerator = new SourceGenerator(parser, generatorPayload); + sourceGenerator.setDBElement(TestPatchUtil.getColumnNameMap().get(getNameValue().getName())); + this(esDsm, propertyName, generatorPayload, sourceGenerator); } - @Override - protected DBElement getDBElement() { - return TestPatchUtil.getColumnNameMap().get(getNameValue().getName()); + private TestCollectionProcessor(ESDsm esDsm, String propertyName, GeneratorPayload generatorPayload, + Collector collector) { + super(esDsm, propertyName, generatorPayload, collector); } + +// @Override +// protected DBElement getDBElement() { +// return TestPatchUtil.getColumnNameMap().get(getNameValue().getName()); +// } } } From 8e15f691d0ba49f034eed90da56e506e66d09c6d Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 29 Oct 2021 19:55:59 +0400 Subject: [PATCH 077/405] DDP-7047 | fix collection processor test --- .../export/generate/BaseGenerator.java | 2 +- .../export/process/CollectionProcessor.java | 4 --- .../process/CollectionProcessorTest.java | 29 +++++++++---------- 3 files changed, 15 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java index a5266b6cc..cc8cffd97 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java @@ -27,7 +27,7 @@ public BaseGenerator(Parser parser, GeneratorPayload generatorPayload) { dbElement = Util.getDBElement(getNameValue().getName()); } - //setter method to set dbElement for testing + //setter method to set dbElement for testing only!!! public void setDBElement(DBElement dbElement) { this.dbElement = dbElement; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java index ec9c70e81..85602868b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java @@ -29,10 +29,6 @@ public CollectionProcessor(ESDsm esDsm, String propertyName, GeneratorPayload ge this.collector = collector; } - protected CollectionProcessor() { - - } - @Override public List> process() { List> fetchedRecords = extractDataByReflection(); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorTest.java index 25495e2b4..805e96e35 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorTest.java @@ -6,6 +6,7 @@ import org.broadinstitute.dsm.model.elastic.ESDsm; import org.broadinstitute.dsm.model.elastic.export.TestPatchUtil; import org.broadinstitute.dsm.model.elastic.export.generate.Collector; +import org.broadinstitute.dsm.model.elastic.export.generate.Generator; import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; import org.broadinstitute.dsm.model.elastic.export.generate.SourceGenerator; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; @@ -34,8 +35,8 @@ public void testProcess() throws IOException { NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "mr_updated"); GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, (int)recordId); - ValueParser valueParser = new ValueParser(); - CollectionProcessor collectionProcessor = new TestCollectionProcessor(esDsm, propertyName, valueParser, generatorPayload); + Processor collectionProcessor = new TestCollectionProcessor(esDsm, propertyName, generatorPayload, instance(generatorPayload, + nameValue)); List> updatedList = collectionProcessor.process(); @@ -59,7 +60,8 @@ public void updateIfExistsOrPut() throws IOException { NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "val"); GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, 10); - CollectionProcessor collectionProcessor = new TestCollectionProcessor(esDsm, propertyName, new ValueParser(), generatorPayload); + CollectionProcessor collectionProcessor = new TestCollectionProcessor(esDsm, propertyName, generatorPayload, + instance(generatorPayload, nameValue)); List> updatedList = collectionProcessor.process(); @@ -80,7 +82,8 @@ public void updateIfExists() throws IOException { NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "{\"TEST1\":\"TEST_VAL\", \"TEST2\":\"TEST_VAL3\"}"); GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, 5); - CollectionProcessor collectionProcessor = new TestCollectionProcessor(esDsm, propertyName, new ValueParser(), generatorPayload); + CollectionProcessor collectionProcessor = new TestCollectionProcessor(esDsm, propertyName, generatorPayload, + instance(generatorPayload, nameValue)); List> updatedList = collectionProcessor.process(); @@ -95,23 +98,19 @@ public void updateIfExists() throws IOException { .ifPresentOrElse(m -> m.get("TEST2").equals("TEST_VAL3"), Assert::fail); } + private Collector instance(GeneratorPayload generatorPayload, NameValue nameValue) { + SourceGenerator sourceGenerator = new SourceGenerator(new ValueParser(), generatorPayload); + sourceGenerator.setDBElement(TestPatchUtil.getColumnNameMap().get(nameValue.getName())); + return sourceGenerator; + } + private static class TestCollectionProcessor extends CollectionProcessor { - public TestCollectionProcessor(ESDsm esDsm, String propertyName, Parser parser, GeneratorPayload generatorPayload) { - SourceGenerator sourceGenerator = new SourceGenerator(parser, generatorPayload); - sourceGenerator.setDBElement(TestPatchUtil.getColumnNameMap().get(getNameValue().getName())); - this(esDsm, propertyName, generatorPayload, sourceGenerator); - } - private TestCollectionProcessor(ESDsm esDsm, String propertyName, GeneratorPayload generatorPayload, - Collector collector) { + public TestCollectionProcessor(ESDsm esDsm, String propertyName, GeneratorPayload generatorPayload, Collector collector) { super(esDsm, propertyName, generatorPayload, collector); } -// @Override -// protected DBElement getDBElement() { -// return TestPatchUtil.getColumnNameMap().get(getNameValue().getName()); -// } } } From fe5f9947ce053a02d20f08377ee9ada6e154e023 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 1 Nov 2021 16:53:54 +0400 Subject: [PATCH 078/405] DDP-7047 | try to extract template method --- .../export/process/CollectionProcessor.java | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java index 85602868b..d7ae8d3ff 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java @@ -67,6 +67,15 @@ private List> updateIfExistsOrPut(List> return fetchedRecords; } + protected void templateMethod(List> records) { + Object collectedData = collector.collect(); + if (collectedData instanceof Map) { + processMap(records); + } else { + processList(records); + } + } + private void addNewRecordTo(List> fetchedRecords) { Object collectedData = collector.collect(); if (collectedData instanceof Map) { @@ -79,6 +88,22 @@ private void addNewRecordTo(List> fetchedRecords) { } } + private void upsertRecord(List> fetchedRecords) { + Object collectedData = collector.collect(); + if (collectedData instanceof Map) { + Map recordMap = (Map) collectedData; + recordMap.put(MappingGenerator.ID, generatorPayload.getRecordId()); + fetchedRecords.add(recordMap); + eachRecord.putAll(recordMap); + } else { + for (Map entry: fetchedRecords) { + + } + List> records = (List>) collectedData; + records.forEach(eachRecord::putAll); //eachRecord.putAll(Map) + } + } + private boolean isExistingRecord(Map eachRecord) { return (double) eachRecord.get(BaseGenerator.ID) == (double) generatorPayload.getRecordId(); } @@ -91,7 +116,9 @@ private void updateExistingRecord(Map eachRecord) { eachRecord.putAll(recordMap); } else { List> records = (List>) collectedData; - records.forEach(eachRecord::putAll); + records.forEach(eachRecord::putAll); //eachRecord.putAll(Map) } } + + } From 8cec0cd2b211316002d749acb5f39990924522f8 Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 2 Nov 2021 11:12:49 +0400 Subject: [PATCH 079/405] DDP-7047 | tweak code, remove duplication, refactor a bit, start adding loggers --- .../elastic/export/ElasticExportAdapter.java | 10 ++- .../model/elastic/export/ExportFacade.java | 7 +- .../export/generate/MappingGenerator.java | 15 +++-- .../export/generate/SourceGenerator.java | 8 ++- .../elastic/export/parse/TypeParser.java | 20 ++++-- .../export/process/CollectionProcessor.java | 65 +++---------------- .../dsm/model/patch/AbstractionPatch.java | 1 - .../export/generate/MappingGeneratorTest.java | 31 ++++++--- 8 files changed, 79 insertions(+), 78 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticExportAdapter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticExportAdapter.java index 824250583..d68d844fb 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticExportAdapter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticExportAdapter.java @@ -4,30 +4,36 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.indices.PutMappingRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.Map; public class ElasticExportAdapter extends BaseExporter { + private static final Logger logger = LoggerFactory.getLogger(ElasticExportAdapter.class); + @Override public void exportData(Map data) { UpdateRequest updateRequest = upsertDataRequestPayload.getUpdateRequest(data); try { clientInstance.update(updateRequest, RequestOptions.DEFAULT); } catch (IOException e) { - throw new RuntimeException("Error while exporting data to ES", e); + throw new RuntimeException("Error occurred while exporting data to ES", e); } } @Override public void exportMapping(Map mapping) { + logger.info("initialize exporting mapping to ES"); PutMappingRequest putMappingRequest = upsertMappingRequestPayload.getPutMappingRequest(); putMappingRequest.source(mapping); try { clientInstance.indices().putMapping(putMappingRequest, RequestOptions.DEFAULT); } catch (IOException e) { - throw new RuntimeException("Error while updating mapping to ES", e); + throw new RuntimeException("Error occurred while updating mapping to ES", e); } + logger.info("successfully exported mapping to ES"); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 8caed38f8..42d864068 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -19,9 +19,13 @@ import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchable; import org.broadinstitute.dsm.util.PatchUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class ExportFacade { + private static final Logger logger = LoggerFactory.getLogger(ExportFacade.class); + BaseExporter exportable; Generator generator; ElasticSearchable searchable; @@ -59,7 +63,8 @@ private Map processData(ESDsm esDsm) { ValueParser valueParser = new ValueParser(); SourceGenerator sourceGenerator = new SourceGenerator(valueParser, exportFacadePayload.getGeneratorPayload()); this.generator = sourceGenerator; - Map dataToReturn = this.generator.generate(); + Map dataToReturn = generator.generate(); + logger.info("processing ES participant data"); if (propertyInfo.isCollection()) { processor = new CollectionProcessor(esDsm, propertyInfo.getPropertyName(), exportFacadePayload.getGeneratorPayload(), sourceGenerator); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java index 82bafae50..26f98e24b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java @@ -1,12 +1,16 @@ package org.broadinstitute.dsm.model.elastic.export.generate; +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.util.HashMap; import java.util.Map; -import org.broadinstitute.dsm.model.elastic.export.parse.Parser; - public class MappingGenerator extends BaseGenerator { + private static final Logger logger = LoggerFactory.getLogger(MappingGenerator.class); + public static final String TYPE = "type"; public static final String NESTED = "nested"; public static final String TYPE_KEYWORD = "keyword"; @@ -18,6 +22,7 @@ public MappingGenerator(Parser parser, GeneratorPayload generatorPayload) { @Override public Map generate() { + logger.info("preparing mapping to upsert"); String propertyName = getOuterPropertyByAlias().getPropertyName(); Map mappedField = buildMappedField(); Map objectLevel = Map.of(propertyName, mappedField); @@ -36,7 +41,7 @@ protected Map parseJson() { Map fieldsByValues = parseJsonToMapFromValue(); for (Map.Entry entry: fieldsByValues.entrySet()) { Object eachType = parser.parse(String.valueOf(entry.getValue())); - resultMap.put(entry.getKey(), Map.of(MappingGenerator.TYPE, eachType)); + resultMap.put(entry.getKey(), eachType); } return resultMap; } @@ -50,13 +55,13 @@ protected Object parseSingleElement() { protected Map getElementWithId(Object type) { return Map.of( ID, Map.of(TYPE, TYPE_KEYWORD), - getDBElement().getColumnName(), Map.of(MappingGenerator.TYPE, type) + getDBElement().getColumnName(), type ); } @Override protected Map getElement(Object type) { - return Map.of(getDBElement().getColumnName(), Map.of(MappingGenerator.TYPE, type)); + return Map.of(getDBElement().getColumnName(), type); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java index d526a6fae..a498cd707 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java @@ -1,13 +1,16 @@ package org.broadinstitute.dsm.model.elastic.export.generate; +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.util.HashMap; import java.util.List; import java.util.Map; -import org.broadinstitute.dsm.model.elastic.export.parse.Parser; - public class SourceGenerator extends BaseGenerator { + private static final Logger logger = LoggerFactory.getLogger(SourceGenerator.class); public SourceGenerator(Parser parser, GeneratorPayload generatorPayload) { super(parser, generatorPayload); @@ -16,6 +19,7 @@ public SourceGenerator(Parser parser, GeneratorPayload generatorPayload) { @Override public Map generate() { + logger.info(""); Object dataToExport = collect(); Map objectLevel = Map.of(getOuterPropertyByAlias().getPropertyName(), dataToExport); return Map.of(DSM_OBJECT, objectLevel); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java index 90ecfcc8b..b205ed36a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java @@ -1,25 +1,37 @@ package org.broadinstitute.dsm.model.elastic.export.parse; +import org.broadinstitute.dsm.model.elastic.export.generate.MappingGenerator; + +import java.util.Map; + public class TypeParser extends BaseParser { + private static final String TEXT = "text"; + private static final String FIELDS = "fields"; + private static final String KEYWORD = "keyword"; + private static final String TYPE = "type"; + public static final Map TEXT_KEYWORD_MAPPING = Map.of(TYPE, TEXT, FIELDS, Map.of(KEYWORD, Map.of(TYPE, KEYWORD))); + private static final String BOOLEAN = "boolean"; + private static final String DATE = "date"; + @Override protected Object forNumeric(String value) { - return "long"; + return TEXT_KEYWORD_MAPPING; } @Override protected Object forBoolean(String value) { - return "boolean"; + return Map.of(MappingGenerator.TYPE, BOOLEAN); } @Override protected Object forDate(String value) { - return "date"; + return Map.of(MappingGenerator.TYPE, DATE); } @Override protected Object forString(String value) { - return "text"; + return TEXT_KEYWORD_MAPPING; } } \ No newline at end of file diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java index d7ae8d3ff..d42d5620a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java @@ -4,14 +4,10 @@ import org.broadinstitute.dsm.model.elastic.export.generate.Collector; import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; -import org.broadinstitute.dsm.model.elastic.export.generate.MappingGenerator; -import org.broadinstitute.dsm.model.elastic.export.generate.SourceGenerator; -import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import java.lang.reflect.Field; import java.util.*; import java.util.function.Predicate; -import java.util.stream.Collectors; public class CollectionProcessor implements Processor { @@ -56,52 +52,15 @@ private List> getRecordsByField(Field field) { } private List> updateIfExistsOrPut(List> fetchedRecords) { - List> existingRecords = fetchedRecords.stream() + fetchedRecords.stream() .filter(this::isExistingRecord) - .collect(Collectors.toList()); - if (existingRecords.isEmpty()) { - addNewRecordTo(fetchedRecords); - } else { - existingRecords.forEach(this::updateExistingRecord); - } + .findFirst() + .ifPresentOrElse(this::updateExistingRecord, () -> addNewRecordTo(fetchedRecords)); return fetchedRecords; } - protected void templateMethod(List> records) { - Object collectedData = collector.collect(); - if (collectedData instanceof Map) { - processMap(records); - } else { - processList(records); - } - } - private void addNewRecordTo(List> fetchedRecords) { - Object collectedData = collector.collect(); - if (collectedData instanceof Map) { - Map recordMap = (Map) collectedData; - recordMap.put(MappingGenerator.ID, generatorPayload.getRecordId()); - fetchedRecords.add(recordMap); - } else { - List> recordList = (List>) collectedData; - fetchedRecords.addAll(recordList); - } - } - - private void upsertRecord(List> fetchedRecords) { - Object collectedData = collector.collect(); - if (collectedData instanceof Map) { - Map recordMap = (Map) collectedData; - recordMap.put(MappingGenerator.ID, generatorPayload.getRecordId()); - fetchedRecords.add(recordMap); - eachRecord.putAll(recordMap); - } else { - for (Map entry: fetchedRecords) { - - } - List> records = (List>) collectedData; - records.forEach(eachRecord::putAll); //eachRecord.putAll(Map) - } + collectEndResult().ifPresent(fetchedRecords::add); } private boolean isExistingRecord(Map eachRecord) { @@ -109,16 +68,12 @@ private boolean isExistingRecord(Map eachRecord) { } private void updateExistingRecord(Map eachRecord) { - Object collectedData = collector.collect(); - if (collectedData instanceof Map) { - Map recordMap = (Map) collectedData; - recordMap.put(MappingGenerator.ID, generatorPayload.getRecordId()); - eachRecord.putAll(recordMap); - } else { - List> records = (List>) collectedData; - records.forEach(eachRecord::putAll); //eachRecord.putAll(Map) - } + collectEndResult().ifPresent(eachRecord::putAll); } - + private Optional> collectEndResult() { + return ((List>) collector.collect()) + .stream() + .findFirst(); + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java index f64b3c9f3..72e2e160b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/AbstractionPatch.java @@ -1,6 +1,5 @@ package org.broadinstitute.dsm.model.patch; -import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.AbstractionWrapper; import org.broadinstitute.dsm.model.NameValue; import spark.utils.StringUtils; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java index 771adbc65..273537caa 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java @@ -27,27 +27,27 @@ public void generateTextType() { } @Test - public void generateBooleanType() { + public void generateTextTypeWithFields() { GeneratorPayload generatorPayload = new GeneratorPayload( - new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "true"), - 0 + new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "value"), + 0 ); Map objectMap = TestMappingGenerator.of(generatorPayload).generate(); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), BaseGenerator.PROPERTIES); - String type = extractDeepestLeveleValue(objectMap); - Assert.assertEquals("boolean", type); + String type = extractKeywordType(objectMap); + Assert.assertEquals("keyword", type); } @Test - public void generateIntegerType() { + public void generateBooleanType() { GeneratorPayload generatorPayload = new GeneratorPayload( - new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "45"), + new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "true"), 0 ); Map objectMap = TestMappingGenerator.of(generatorPayload).generate(); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), BaseGenerator.PROPERTIES); String type = extractDeepestLeveleValue(objectMap); - Assert.assertEquals("long", type); + Assert.assertEquals("boolean", type); } @Test @@ -113,6 +113,20 @@ private String extractDeepestLeveleValue(Map objectMap) { .get("type"); } + private String extractKeywordType(Map objectMap) { + return (String) + ((Map) + ((Map) + ((Map) + ((Map) + getMedicalRecordProperty(objectMap) + .get(BaseGenerator.PROPERTIES)) + .get("medical_record_column")) + .get("fields")) + .get("keyword")) + .get("type"); + } + private Map getMedicalRecordProperty(Map objectMap) { return (Map) ((Map) @@ -124,6 +138,7 @@ private Map getMedicalRecordProperty(Map objectMap) { .get("medicalRecords"); } + private static class TestMappingGenerator extends MappingGenerator { public TestMappingGenerator(Parser typeParser, GeneratorPayload generatorPayload) { From 04ebaa547822ddd8f9b05868d96571dc4ff93378 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 2 Nov 2021 11:48:31 +0400 Subject: [PATCH 080/405] DDP-7047 | add more logger infos --- .../elastic/export/ElasticExportAdapter.java | 2 ++ .../model/elastic/export/ExportFacade.java | 4 +++- .../export/generate/BaseGenerator.java | 11 ++++++---- .../export/generate/SourceGenerator.java | 22 +++++++++++++++---- .../export/process/CollectionProcessor.java | 22 ++++++++++++++----- 5 files changed, 47 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticExportAdapter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticExportAdapter.java index d68d844fb..cd16a5920 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticExportAdapter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticExportAdapter.java @@ -16,12 +16,14 @@ public class ElasticExportAdapter extends BaseExporter { @Override public void exportData(Map data) { + logger.info("initialize exporting data to ES"); UpdateRequest updateRequest = upsertDataRequestPayload.getUpdateRequest(data); try { clientInstance.update(updateRequest, RequestOptions.DEFAULT); } catch (IOException e) { throw new RuntimeException("Error occurred while exporting data to ES", e); } + logger.info("successfully exported data to ES"); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 42d864068..4c51f2924 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -64,7 +64,7 @@ private Map processData(ESDsm esDsm) { SourceGenerator sourceGenerator = new SourceGenerator(valueParser, exportFacadePayload.getGeneratorPayload()); this.generator = sourceGenerator; Map dataToReturn = generator.generate(); - logger.info("processing ES participant data"); + logger.info("Processing ES participant data"); if (propertyInfo.isCollection()) { processor = new CollectionProcessor(esDsm, propertyInfo.getPropertyName(), exportFacadePayload.getGeneratorPayload(), sourceGenerator); @@ -73,6 +73,7 @@ private Map processData(ESDsm esDsm) { dataToReturn = Map.of(MappingGenerator.DSM_OBJECT, Map.of(propertyInfo.getPropertyName(), processedData)); } } + logger.info("Returning processed ES participant data"); return dataToReturn; } @@ -87,6 +88,7 @@ private void upsertData(Map elasticDataToExport) { .withDocAsUpsert(true) .withRetryOnConflict(5) .build(); + logger.info("Built upsert data request payload"); exportable.setUpdateRequestPayload(upsertDataRequestPayload); exportable.exportData(elasticDataToExport); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java index cc8cffd97..2274e11ea 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java @@ -1,17 +1,20 @@ package org.broadinstitute.dsm.model.elastic.export.generate; +import java.util.Map; +import java.util.Objects; + import com.google.gson.Gson; import com.google.gson.JsonSyntaxException; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; - -import java.util.Map; -import java.util.Objects; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public abstract class BaseGenerator implements Generator, Collector { + private static final Logger logger = LoggerFactory.getLogger(BaseGenerator.class); public static final String DSM_OBJECT = "dsm"; public static final String PROPERTIES = "properties"; @@ -81,7 +84,7 @@ protected Object getFieldWithElement() { protected Object constructByPropertyType() { Object constructedObject; - if(getOuterPropertyByAlias().isCollection()) { + if (getOuterPropertyByAlias().isCollection()) { constructedObject = constructCollection(); } else { constructedObject = constructSingleElement(); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java index a498cd707..6278fef9a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java @@ -19,10 +19,13 @@ public SourceGenerator(Parser parser, GeneratorPayload generatorPayload) { @Override public Map generate() { - logger.info(""); Object dataToExport = collect(); - Map objectLevel = Map.of(getOuterPropertyByAlias().getPropertyName(), dataToExport); - return Map.of(DSM_OBJECT, objectLevel); + logger.info("Generating final source"); + return Map.of(DSM_OBJECT, buildPropertyLevelWithData(dataToExport)); + } + + private Map buildPropertyLevelWithData(Object dataToExport) { + return Map.of(getOuterPropertyByAlias().getPropertyName(), dataToExport); } @Override @@ -31,6 +34,7 @@ protected Object parseJson() { } private Map parseJsonValuesToObject() { + logger.info("Converting JSON values to Map"); Map dynamicFieldValues = parseJsonToMapFromValue(); Map transformedMap = new HashMap<>(); for (Map.Entry entry : dynamicFieldValues.entrySet()) { @@ -46,12 +50,21 @@ protected Object parseSingleElement() { @Override protected Object getElementWithId(Object element) { - return List.of(Map.of(getDBElement().getColumnName(),element, + return buildCollectionWithId(element); + } + + private List> buildCollectionWithId(Object element) { + return List.of(Map.of(getDBElement().getColumnName(), element, ID, generatorPayload.getRecordId())); } @Override protected Map getElement(Object element) { + return buildSingleFieldWithValue(element); + } + + private Map buildSingleFieldWithValue(Object element) { + logger.info("Constructing single field with value"); return Map.of(getDBElement().getColumnName(), element); } @@ -62,6 +75,7 @@ protected Object constructSingleElement() { @Override protected Object constructCollection() { + logger.info("Constructing nested data"); Map collectionMap = new HashMap<>(); collectionMap.put(ID, generatorPayload.getRecordId()); Map mapWithParsedObjects = parseJsonValuesToObject(); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java index d42d5620a..5e0aae69f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java @@ -1,16 +1,25 @@ package org.broadinstitute.dsm.model.elastic.export.process; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.function.Predicate; + import org.broadinstitute.dsm.model.elastic.ESDsm; +import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; import org.broadinstitute.dsm.model.elastic.export.generate.Collector; import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; -import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; - -import java.lang.reflect.Field; -import java.util.*; -import java.util.function.Predicate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class CollectionProcessor implements Processor { + private static final Logger logger = LoggerFactory.getLogger(CollectionProcessor.class); + private ESDsm esDsm; private String propertyName; private GeneratorPayload generatorPayload; @@ -32,6 +41,7 @@ public List> process() { } private List> extractDataByReflection() { + logger.info("Extracting data by field from fetched ES data"); Field[] declaredFields = esDsm.getClass().getDeclaredFields(); List> fetchedRecords = Arrays.stream(declaredFields).filter(isFieldMatchProperty) .findFirst() @@ -60,6 +70,7 @@ private List> updateIfExistsOrPut(List> } private void addNewRecordTo(List> fetchedRecords) { + logger.info("Adding new record"); collectEndResult().ifPresent(fetchedRecords::add); } @@ -68,6 +79,7 @@ private boolean isExistingRecord(Map eachRecord) { } private void updateExistingRecord(Map eachRecord) { + logger.info("Updating existing record"); collectEndResult().ifPresent(eachRecord::putAll); } From b3d72b434943b9577f518b998d55e7b5e2150a8b Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 2 Nov 2021 12:01:42 +0400 Subject: [PATCH 081/405] DDP-7047 | extract local strings as final constants --- .../dsm/db/structure/DBElement.java | 2 -- .../broadinstitute/dsm/model/elastic/Util.java | 16 ++++++++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/structure/DBElement.java b/src/main/java/org/broadinstitute/dsm/db/structure/DBElement.java index 1d0dc8a9f..d27f46456 100644 --- a/src/main/java/org/broadinstitute/dsm/db/structure/DBElement.java +++ b/src/main/java/org/broadinstitute/dsm/db/structure/DBElement.java @@ -11,8 +11,6 @@ @Data public class DBElement { - public static final List JSON_TYPE_COLUMNS = List.of("data", "something", "another thing"); - public String tableName; public String tableAlias; public String primaryKey; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 983fe8a67..9166cca88 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -24,13 +24,13 @@ public class Util { public static String getQueryTypeFromId(String id) { String type; if (ParticipantUtil.isHruid(id)) { - type = "profile.hruid"; + type = Constants.PROFILE_HRUID; } else if (ParticipantUtil.isGuid(id)){ - type = "profile.guid"; + type = Constants.PROFILE_GUID; } else if (ParticipantUtil.isLegacyAltPid(id)) { - type = "profile.legacyAltPid"; + type = Constants.PROFILE_LEGACYALTPID; } else { - type = "profile.legacyShortId"; + type = Constants.PROFILE_LEGACYSHORTID; } return type; } @@ -38,4 +38,12 @@ public static String getQueryTypeFromId(String id) { public static DBElement getDBElement(String fieldName) { return PatchUtil.getColumnNameMap().get(Objects.requireNonNull(fieldName)); } + + public static class Constants { + public static final String PROFILE = "profile"; + public static final String PROFILE_HRUID = PROFILE + ".hruid"; + public static final String PROFILE_GUID = PROFILE + ".guid"; + public static final String PROFILE_LEGACYALTPID = PROFILE + ".legacyAltPid"; + public static final String PROFILE_LEGACYSHORTID = PROFILE + ".legacyShortId"; + } } From 2585d83aeffa843f30498f0cfb68e7b444931bc4 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 4 Nov 2021 10:44:08 +0400 Subject: [PATCH 082/405] DDP-7117 | start creating of medical record mapping --- .../dsm/model/elastic/Util.java | 18 +++++ .../elastic/export/parse/TypeParser.java | 6 +- .../migrationscript/MedicalRecordMigrate.java | 79 +++++++++++++++++++ .../dsm/model/elastic/UtilTest.java | 24 ++++++ .../MedicalRecordMigrateTest.java | 28 +++++++ 5 files changed, 153 insertions(+), 2 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 9166cca88..14358fa7a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -1,7 +1,10 @@ package org.broadinstitute.dsm.model.elastic; +import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; @@ -20,6 +23,8 @@ public class Util { "p", new BaseGenerator.PropertyInfo(ESObjectConstants.PARTICIPANT, false), "o", new BaseGenerator.PropertyInfo(ESObjectConstants.ONC_HISTORY, false) ); + public static final int FIRST_ELEMENT_INDEX = 0; + public static final String UNDERSCORE_SEPARATOR = "_"; public static String getQueryTypeFromId(String id) { String type; @@ -39,6 +44,19 @@ public static DBElement getDBElement(String fieldName) { return PatchUtil.getColumnNameMap().get(Objects.requireNonNull(fieldName)); } + public static String underscoresToCamelCase(String fieldName) { + List words = Arrays.stream(fieldName.split(UNDERSCORE_SEPARATOR)) + .map(word -> new StringBuilder(word.toLowerCase())) + .collect(Collectors.toList()); + for (int i = FIRST_ELEMENT_INDEX; i < words.size(); i++) { + StringBuilder word = words.get(i); + if (i != FIRST_ELEMENT_INDEX && word.length() > FIRST_ELEMENT_INDEX) { + word.replace(FIRST_ELEMENT_INDEX, 1, String.valueOf(word.charAt(FIRST_ELEMENT_INDEX)).toUpperCase()); + } + } + return String.join("", words); + } + public static class Constants { public static final String PROFILE = "profile"; public static final String PROFILE_HRUID = PROFILE + ".hruid"; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java index b205ed36a..9e316cb4f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java @@ -13,7 +13,9 @@ public class TypeParser extends BaseParser { private static final String TYPE = "type"; public static final Map TEXT_KEYWORD_MAPPING = Map.of(TYPE, TEXT, FIELDS, Map.of(KEYWORD, Map.of(TYPE, KEYWORD))); private static final String BOOLEAN = "boolean"; + public static final Map BOOLEAN_MAPPING = Map.of(MappingGenerator.TYPE, BOOLEAN); private static final String DATE = "date"; + public static final Map DATE_MAPPING = Map.of(MappingGenerator.TYPE, DATE); @Override protected Object forNumeric(String value) { @@ -22,12 +24,12 @@ protected Object forNumeric(String value) { @Override protected Object forBoolean(String value) { - return Map.of(MappingGenerator.TYPE, BOOLEAN); + return BOOLEAN_MAPPING; } @Override protected Object forDate(String value) { - return Map.of(MappingGenerator.TYPE, DATE); + return DATE_MAPPING; } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java new file mode 100644 index 000000000..0c68a14fa --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java @@ -0,0 +1,79 @@ +package org.broadinstitute.dsm.model.elastic.migrationscript; + +import java.lang.annotation.Annotation; +import java.lang.reflect.AccessibleObject; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import org.broadinstitute.dsm.db.MedicalRecord; +import org.broadinstitute.dsm.db.structure.ColumnName; +import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; +import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.*; + +public class MedicalRecordMigrate { + + + private static final Map medicalRecordMapping = Map.of( + "ddpParticipantId", TEXT_KEYWORD_MAPPING, + "ddpInstanceId", TEXT_KEYWORD_MAPPING, + "institutionId", TEXT_KEYWORD_MAPPING, + "ddpInstitutionId", TEXT_KEYWORD_MAPPING, + "type", TEXT_KEYWORD_MAPPING, + "participantId", TEXT_KEYWORD_MAPPING, + "medicalRecordId", TEXT_KEYWORD_MAPPING, + "name", TEXT_KEYWORD_MAPPING, + "contact", TEXT_KEYWORD_MAPPING, + "phone", TEXT_KEYWORD_MAPPING, + "fax", TEXT_KEYWORD_MAPPING, + "faxSent", DATE_MAPPING, //DATE? + "faxSentBy", TEXT_KEYWORD_MAPPING, + "faxConfirmed", DATE_MAPPING, //DATE? + "faxSent2", DATE_MAPPING, //DATE? + "faxSent2By", TEXT_KEYWORD_MAPPING, + "faxConfirmed2", DATE_MAPPING, //DATE? + "faxSent3", DATE_MAPPING, //DATE? + "faxSent3By", TEXT_KEYWORD_MAPPING, + "faxConfirmed3", DATE_MAPPING, //DATE? + "mrReceived", DATE_MAPPING, //DATE? + "followUps", TEXT_KEYWORD_MAPPING, + "mrDocument", TEXT_KEYWORD_MAPPING, + "mrDocumentFileName", TEXT_KEYWORD_MAPPING, + "mrProblem", BOOLEAN_MAPPING, + "mrProblemText", TEXT_KEYWORD_MAPPING, + "", TEXT_KEYWORD_MAPPING, + ) + + + + protected List collectMedicalRecordColumns() { + Class medicalRecordClass = MedicalRecord.class; + Field[] fields = medicalRecordClass.getDeclaredFields(); + + List columnNames = Arrays.stream(fields) + .map(AccessibleObject::getAnnotations) + .map(annotations -> Arrays.stream(annotations).filter(this::isColumnNameType).findFirst()) + .filter(Optional::isPresent) + .map(Optional::get) + .map(annotation -> ((ColumnName) annotation).value()) + .collect(Collectors.toList()); + return columnNames; + } + + protected List swapToCamelCases(List columnNames) { + return columnNames.stream() + .map(Util::underscoresToCamelCase) + .collect(Collectors.toList()); + } + + private boolean isColumnNameType(Annotation annotation) { + return annotation instanceof ColumnName; + } +} diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java new file mode 100644 index 000000000..ef609f135 --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java @@ -0,0 +1,24 @@ +package org.broadinstitute.dsm.model.elastic; + +import static org.junit.Assert.*; + +import org.junit.Test; + +public class UtilTest { + + @Test + public void underscoresToCamelCase() { + String fieldName = "column_name"; + String fieldName2 = "COLUMN_NAME"; + String fieldName3 = "column"; + String fieldName4 = "COLUMN"; + String transformed = Util.underscoresToCamelCase(fieldName); + String transformed2 = Util.underscoresToCamelCase(fieldName2); + String transformed3 = Util.underscoresToCamelCase(fieldName3); + String transformed4 = Util.underscoresToCamelCase(fieldName4); + assertEquals("columnName", transformed); + assertEquals("columnName", transformed2); + assertEquals("column", transformed3); + assertEquals("column", transformed4); + } +} \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java new file mode 100644 index 000000000..3e2e369d7 --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java @@ -0,0 +1,28 @@ +package org.broadinstitute.dsm.model.elastic.migrationscript; + +import static org.junit.Assert.*; + +import java.util.Arrays; +import java.util.List; + +import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.statics.DBConstants; +import org.junit.Test; + +public class MedicalRecordMigrateTest { + + @Test + public void collectMedicalRecordColumns() { + MedicalRecordMigrate medicalRecordMigrate = new MedicalRecordMigrate(); + List columnNames = medicalRecordMigrate.collectMedicalRecordColumns(); + assertTrue(columnNames.contains(DBConstants.MR_RECEIVED)); + } + + @Test + public void swapToCamelCases() { + MedicalRecordMigrate medicalRecordMigrate = new MedicalRecordMigrate(); + List columnNames = medicalRecordMigrate.collectMedicalRecordColumns(); + List camelCaseColumns = medicalRecordMigrate.swapToCamelCases(columnNames); + assertEquals("mrReceived", camelCaseColumns.stream().filter("mrReceived"::equals).findFirst().get()); + } +} \ No newline at end of file From 21eaa738b645d0f70b1ef203102ea4417c749a3a Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 4 Nov 2021 11:09:22 +0400 Subject: [PATCH 083/405] DDP-7117 | finish MedicalRecordMigrate and initialize OncHistoryDetailMigrate --- .../migrationscript/MedicalRecordMigrate.java | 41 ++++++++---- .../OncHistoryDetailsMigrate.java | 65 +++++++++++++++++++ 2 files changed, 95 insertions(+), 11 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java index 0c68a14fa..a3f181d1b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java @@ -3,11 +3,7 @@ import java.lang.annotation.Annotation; import java.lang.reflect.AccessibleObject; import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Optional; +import java.util.*; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -21,8 +17,8 @@ public class MedicalRecordMigrate { - private static final Map medicalRecordMapping = Map.of( - "ddpParticipantId", TEXT_KEYWORD_MAPPING, + private static final Map medicalRecordMapping1 = Map.of ( + "ddpParticipantId", TEXT_KEYWORD_MAPPING, "ddpInstanceId", TEXT_KEYWORD_MAPPING, "institutionId", TEXT_KEYWORD_MAPPING, "ddpInstitutionId", TEXT_KEYWORD_MAPPING, @@ -30,7 +26,9 @@ public class MedicalRecordMigrate { "participantId", TEXT_KEYWORD_MAPPING, "medicalRecordId", TEXT_KEYWORD_MAPPING, "name", TEXT_KEYWORD_MAPPING, - "contact", TEXT_KEYWORD_MAPPING, + "contact", TEXT_KEYWORD_MAPPING ); + + private static final Map medicalRecordMapping2 = Map.of ( "phone", TEXT_KEYWORD_MAPPING, "fax", TEXT_KEYWORD_MAPPING, "faxSent", DATE_MAPPING, //DATE? @@ -40,7 +38,9 @@ public class MedicalRecordMigrate { "faxSent2By", TEXT_KEYWORD_MAPPING, "faxConfirmed2", DATE_MAPPING, //DATE? "faxSent3", DATE_MAPPING, //DATE? - "faxSent3By", TEXT_KEYWORD_MAPPING, + "faxSent3By", TEXT_KEYWORD_MAPPING); + + private static final Map medicalRecordMapping3 = Map.of ( "faxConfirmed3", DATE_MAPPING, //DATE? "mrReceived", DATE_MAPPING, //DATE? "followUps", TEXT_KEYWORD_MAPPING, @@ -48,10 +48,29 @@ public class MedicalRecordMigrate { "mrDocumentFileName", TEXT_KEYWORD_MAPPING, "mrProblem", BOOLEAN_MAPPING, "mrProblemText", TEXT_KEYWORD_MAPPING, - "", TEXT_KEYWORD_MAPPING, - ) + "unableObtain", BOOLEAN_MAPPING, + "unableObtainText", TEXT_KEYWORD_MAPPING, + "duplicate", BOOLEAN_MAPPING); + private static final Map medicalRecordMapping4 = Map.of ( + "followUpRequired", BOOLEAN_MAPPING, + "followUpRequiredText", TEXT_KEYWORD_MAPPING, + "international", BOOLEAN_MAPPING, + "crRequired", BOOLEAN_MAPPING, + "pathologyPresent", TEXT_KEYWORD_MAPPING, + "notes", TEXT_KEYWORD_MAPPING, + "additionalValuesJson", TEXT_KEYWORD_MAPPING, + "reviewMedicalRecord", TEXT_KEYWORD_MAPPING + ); + private static final Map medicalRecordMappingMerged = new HashMap<>(); + + static { + medicalRecordMappingMerged.putAll(medicalRecordMapping1); + medicalRecordMappingMerged.putAll(medicalRecordMapping2); + medicalRecordMappingMerged.putAll(medicalRecordMapping3); + medicalRecordMappingMerged.putAll(medicalRecordMapping4); + } protected List collectMedicalRecordColumns() { Class medicalRecordClass = MedicalRecord.class; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java new file mode 100644 index 000000000..fa0513c95 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java @@ -0,0 +1,65 @@ +package org.broadinstitute.dsm.model.elastic.migrationscript; + +import java.util.HashMap; +import java.util.Map; + +import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.*; +import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.TEXT_KEYWORD_MAPPING; + +public class OncHistoryDetailsMigrate { + + private static final Map oncHistoryDetails1 = Map.of ( + "ddpParticipantId", TEXT_KEYWORD_MAPPING, + "ddpInstanceId", TEXT_KEYWORD_MAPPING, + "institutionId", TEXT_KEYWORD_MAPPING, + "ddpInstitutionId", TEXT_KEYWORD_MAPPING, + "type", TEXT_KEYWORD_MAPPING, + "participantId", TEXT_KEYWORD_MAPPING, + "medicalRecordId", TEXT_KEYWORD_MAPPING, + "name", TEXT_KEYWORD_MAPPING, + "contact", TEXT_KEYWORD_MAPPING ); + + private static final Map oncHistoryDetails2 = Map.of ( + "phone", TEXT_KEYWORD_MAPPING, + "fax", TEXT_KEYWORD_MAPPING, + "faxSent", DATE_MAPPING, //DATE? + "faxSentBy", TEXT_KEYWORD_MAPPING, + "faxConfirmed", DATE_MAPPING, //DATE? + "faxSent2", DATE_MAPPING, //DATE? + "faxSent2By", TEXT_KEYWORD_MAPPING, + "faxConfirmed2", DATE_MAPPING, //DATE? + "faxSent3", DATE_MAPPING, //DATE? + "faxSent3By", TEXT_KEYWORD_MAPPING); + + private static final Map oncHistoryDetails3 = Map.of ( + "faxConfirmed3", DATE_MAPPING, //DATE? + "mrReceived", DATE_MAPPING, //DATE? + "followUps", TEXT_KEYWORD_MAPPING, + "mrDocument", TEXT_KEYWORD_MAPPING, + "mrDocumentFileName", TEXT_KEYWORD_MAPPING, + "mrProblem", BOOLEAN_MAPPING, + "mrProblemText", TEXT_KEYWORD_MAPPING, + "unableObtain", BOOLEAN_MAPPING, + "unableObtainText", TEXT_KEYWORD_MAPPING, + "duplicate", BOOLEAN_MAPPING); + + private static final Map oncHistoryDetails4 = Map.of ( + "followUpRequired", BOOLEAN_MAPPING, + "followUpRequiredText", TEXT_KEYWORD_MAPPING, + "international", BOOLEAN_MAPPING, + "crRequired", BOOLEAN_MAPPING, + "pathologyPresent", TEXT_KEYWORD_MAPPING, + "notes", TEXT_KEYWORD_MAPPING, + "additionalValuesJson", TEXT_KEYWORD_MAPPING, + "reviewMedicalRecord", TEXT_KEYWORD_MAPPING + ); + + private static final Map oncHistoryDetailsMerged = new HashMap<>(); + + static { + oncHistoryDetailsMerged.putAll(oncHistoryDetails1); + oncHistoryDetailsMerged.putAll(oncHistoryDetails2); + oncHistoryDetailsMerged.putAll(oncHistoryDetails3); + oncHistoryDetailsMerged.putAll(oncHistoryDetails4); + } +} From 6cb836758723010fe06bda57031dc014c63b8134 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 4 Nov 2021 11:45:27 +0400 Subject: [PATCH 084/405] DDP-7117 | start creation of onc history details map, add utility method for collecting mappings --- .../elastic/migrationscript/MapAdapter.java | 16 +++++++ .../migrationscript/MedicalRecordMigrate.java | 10 ++--- .../OncHistoryDetailsMigrate.java | 45 ++++++++++--------- .../MedicalRecordMigrateTest.java | 8 ++++ 4 files changed, 50 insertions(+), 29 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MapAdapter.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MapAdapter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MapAdapter.java new file mode 100644 index 000000000..ecd902947 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MapAdapter.java @@ -0,0 +1,16 @@ +package org.broadinstitute.dsm.model.elastic.migrationscript; + +import java.util.HashMap; +import java.util.Map; + +public class MapAdapter { + + public static Map of(Map... maps) { + Map result = new HashMap<>(); + for (Map map: maps) { + result.putAll(map); + } + return result; + } + +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java index a3f181d1b..bcc126910 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java @@ -63,14 +63,10 @@ public class MedicalRecordMigrate { "reviewMedicalRecord", TEXT_KEYWORD_MAPPING ); - private static final Map medicalRecordMappingMerged = new HashMap<>(); + protected static final Map medicalRecordMappingMerged = MapAdapter.of(medicalRecordMapping1, medicalRecordMapping2, + medicalRecordMapping3, + medicalRecordMapping4);; - static { - medicalRecordMappingMerged.putAll(medicalRecordMapping1); - medicalRecordMappingMerged.putAll(medicalRecordMapping2); - medicalRecordMappingMerged.putAll(medicalRecordMapping3); - medicalRecordMappingMerged.putAll(medicalRecordMapping4); - } protected List collectMedicalRecordColumns() { Class medicalRecordClass = MedicalRecord.class; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java index fa0513c95..706f8b96c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java @@ -10,35 +10,36 @@ public class OncHistoryDetailsMigrate { private static final Map oncHistoryDetails1 = Map.of ( "ddpParticipantId", TEXT_KEYWORD_MAPPING, - "ddpInstanceId", TEXT_KEYWORD_MAPPING, - "institutionId", TEXT_KEYWORD_MAPPING, - "ddpInstitutionId", TEXT_KEYWORD_MAPPING, - "type", TEXT_KEYWORD_MAPPING, - "participantId", TEXT_KEYWORD_MAPPING, + "oncHistoryDetailId", TEXT_KEYWORD_MAPPING, + "request", TEXT_KEYWORD_MAPPING, + "deleted", BOOLEAN_MAPPING, + "faxSent", DATE_MAPPING, //DATE? + "tissueReceived", TEXT_KEYWORD_MAPPING, "medicalRecordId", TEXT_KEYWORD_MAPPING, - "name", TEXT_KEYWORD_MAPPING, - "contact", TEXT_KEYWORD_MAPPING ); + "datePx", DATE_MAPPING, //DATE? + "typePx", TEXT_KEYWORD_MAPPING, + "locationPx", TEXT_KEYWORD_MAPPING ); private static final Map oncHistoryDetails2 = Map.of ( + "histology", TEXT_KEYWORD_MAPPING, + "accessionNumber", TEXT_KEYWORD_MAPPING, + "facility", TEXT_KEYWORD_MAPPING, //DATE? "phone", TEXT_KEYWORD_MAPPING, - "fax", TEXT_KEYWORD_MAPPING, - "faxSent", DATE_MAPPING, //DATE? - "faxSentBy", TEXT_KEYWORD_MAPPING, + "fax", TEXT_KEYWORD_MAPPING, //DATE? + "notes", TEXT_KEYWORD_MAPPING, //DATE? + "additionalValuesJson", TEXT_KEYWORD_MAPPING, + "faxSentBy", TEXT_KEYWORD_MAPPING, //DATE? "faxConfirmed", DATE_MAPPING, //DATE? - "faxSent2", DATE_MAPPING, //DATE? - "faxSent2By", TEXT_KEYWORD_MAPPING, - "faxConfirmed2", DATE_MAPPING, //DATE? - "faxSent3", DATE_MAPPING, //DATE? - "faxSent3By", TEXT_KEYWORD_MAPPING); + "faxSent2", DATE_MAPPING ); //DATE? private static final Map oncHistoryDetails3 = Map.of ( - "faxConfirmed3", DATE_MAPPING, //DATE? - "mrReceived", DATE_MAPPING, //DATE? - "followUps", TEXT_KEYWORD_MAPPING, - "mrDocument", TEXT_KEYWORD_MAPPING, - "mrDocumentFileName", TEXT_KEYWORD_MAPPING, - "mrProblem", BOOLEAN_MAPPING, - "mrProblemText", TEXT_KEYWORD_MAPPING, + "faxSent2By", TEXT_KEYWORD_MAPPING, //DATE? + "faxConfirmed2", DATE_MAPPING, //DATE? + "faxSent3", DATE_MAPPING, + "faxSent3By", TEXT_KEYWORD_MAPPING, + "faxConfirmed3", DATE_MAPPING, + "tissueReceived", TEXT_KEYWORD_MAPPING, + "tissueProblemOption", TEXT_KEYWORD_MAPPING, //HERE "unableObtain", BOOLEAN_MAPPING, "unableObtainText", TEXT_KEYWORD_MAPPING, "duplicate", BOOLEAN_MAPPING); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java index 3e2e369d7..e04b292a9 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java @@ -4,6 +4,7 @@ import java.util.Arrays; import java.util.List; +import java.util.Map; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.statics.DBConstants; @@ -25,4 +26,11 @@ public void swapToCamelCases() { List camelCaseColumns = medicalRecordMigrate.swapToCamelCases(columnNames); assertEquals("mrReceived", camelCaseColumns.stream().filter("mrReceived"::equals).findFirst().get()); } + + @Test + public void medicalRecordMappingMerged() { + Map medicalRecordMappingMerged = + MedicalRecordMigrate.medicalRecordMappingMerged; + assertEquals(37, medicalRecordMappingMerged.size()); + } } \ No newline at end of file From e3e74d8a999565408cbc62ee49988b363ff7decd Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 4 Nov 2021 12:02:49 +0400 Subject: [PATCH 085/405] DDP-7117 | finish OncHistoryDetails --- .../OncHistoryDetailsMigrate.java | 71 ++++++++++++------- 1 file changed, 47 insertions(+), 24 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java index 706f8b96c..6dff29d81 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java @@ -23,44 +23,67 @@ public class OncHistoryDetailsMigrate { private static final Map oncHistoryDetails2 = Map.of ( "histology", TEXT_KEYWORD_MAPPING, "accessionNumber", TEXT_KEYWORD_MAPPING, - "facility", TEXT_KEYWORD_MAPPING, //DATE? + "facility", TEXT_KEYWORD_MAPPING, "phone", TEXT_KEYWORD_MAPPING, - "fax", TEXT_KEYWORD_MAPPING, //DATE? - "notes", TEXT_KEYWORD_MAPPING, //DATE? + "fax", TEXT_KEYWORD_MAPPING, + "notes", TEXT_KEYWORD_MAPPING, "additionalValuesJson", TEXT_KEYWORD_MAPPING, - "faxSentBy", TEXT_KEYWORD_MAPPING, //DATE? + "faxSentBy", TEXT_KEYWORD_MAPPING, "faxConfirmed", DATE_MAPPING, //DATE? "faxSent2", DATE_MAPPING ); //DATE? private static final Map oncHistoryDetails3 = Map.of ( - "faxSent2By", TEXT_KEYWORD_MAPPING, //DATE? + "faxSent2By", TEXT_KEYWORD_MAPPING, "faxConfirmed2", DATE_MAPPING, //DATE? "faxSent3", DATE_MAPPING, "faxSent3By", TEXT_KEYWORD_MAPPING, "faxConfirmed3", DATE_MAPPING, "tissueReceived", TEXT_KEYWORD_MAPPING, - "tissueProblemOption", TEXT_KEYWORD_MAPPING, //HERE - "unableObtain", BOOLEAN_MAPPING, - "unableObtainText", TEXT_KEYWORD_MAPPING, - "duplicate", BOOLEAN_MAPPING); + "tissueProblemOption", TEXT_KEYWORD_MAPPING, + "gender", TEXT_KEYWORD_MAPPING, + "destructionPolicy", TEXT_KEYWORD_MAPPING, + "unableObtainTissue", BOOLEAN_MAPPING); private static final Map oncHistoryDetails4 = Map.of ( - "followUpRequired", BOOLEAN_MAPPING, - "followUpRequiredText", TEXT_KEYWORD_MAPPING, - "international", BOOLEAN_MAPPING, - "crRequired", BOOLEAN_MAPPING, - "pathologyPresent", TEXT_KEYWORD_MAPPING, - "notes", TEXT_KEYWORD_MAPPING, - "additionalValuesJson", TEXT_KEYWORD_MAPPING, - "reviewMedicalRecord", TEXT_KEYWORD_MAPPING + "tissueId", TEXT_KEYWORD_MAPPING, + "countReceived", TEXT_KEYWORD_MAPPING, + "tissueType", TEXT_KEYWORD_MAPPING, + "tissueSite", TEXT_KEYWORD_MAPPING, + "hE", TEXT_KEYWORD_MAPPING, + "pathologyReport", TEXT_KEYWORD_MAPPING, + "collaboratorSampleId", TEXT_KEYWORD_MAPPING, + "blockSent", DATE_MAPPING, // DATE? + "scrollsReceived", DATE_MAPPING, // DATE? + "skId", TEXT_KEYWORD_MAPPING + ); + + private static final Map oncHistoryDetails5 = Map.of ( + "smId", TEXT_KEYWORD_MAPPING, + "sentGp", DATE_MAPPING, // DATE?, + "firstSmId", TEXT_KEYWORD_MAPPING, + "additionalTissueValueJson", TEXT_KEYWORD_MAPPING, + "expectedReturn", DATE_MAPPING, // DATE????????????? + "returnDate", DATE_MAPPING, // DATE? + "returnFedexId", TEXT_KEYWORD_MAPPING, + "shlWorkNumber", TEXT_KEYWORD_MAPPING, + "tumorPercentage", TEXT_KEYWORD_MAPPING, + "tissueSequence", TEXT_KEYWORD_MAPPING + ); + + private static final Map oncHistoryDetails6 = Map.of ( + "scrollsCount", TEXT_KEYWORD_MAPPING, + "ussCount", TEXT_KEYWORD_MAPPING, + "hECount", TEXT_KEYWORD_MAPPING, + "blocksCount", TEXT_KEYWORD_MAPPING ); - private static final Map oncHistoryDetailsMerged = new HashMap<>(); + private static final Map oncHistoryDetailsMerged = MapAdapter.of( + oncHistoryDetails1, + oncHistoryDetails2, + oncHistoryDetails3, + oncHistoryDetails4, + oncHistoryDetails5, + oncHistoryDetails6 + ); - static { - oncHistoryDetailsMerged.putAll(oncHistoryDetails1); - oncHistoryDetailsMerged.putAll(oncHistoryDetails2); - oncHistoryDetailsMerged.putAll(oncHistoryDetails3); - oncHistoryDetailsMerged.putAll(oncHistoryDetails4); - } } From c1b9c419cdfa81105618552df5a2f118b3c95b36 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 4 Nov 2021 16:30:07 +0400 Subject: [PATCH 086/405] DDP-7117 | start creating mapping for kit request shipping --- .../KitRequestShippingMigrate.java | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrate.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrate.java new file mode 100644 index 000000000..5a6f1e8e4 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrate.java @@ -0,0 +1,58 @@ +package org.broadinstitute.dsm.model.elastic.migrationscript; + +import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.BOOLEAN_MAPPING; +import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.DATE_MAPPING; +import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.TEXT_KEYWORD_MAPPING; + +import java.util.Map; + +public class KitRequestShippingMigrate { + + private static final Map kitRequestShippingMigrate1 = Map.of ( + "ddpParticipantId", TEXT_KEYWORD_MAPPING, + "ddpInstanceId", TEXT_KEYWORD_MAPPING, + "institutionId", TEXT_KEYWORD_MAPPING, + "ddpInstitutionId", TEXT_KEYWORD_MAPPING, + "type", TEXT_KEYWORD_MAPPING, + "participantId", TEXT_KEYWORD_MAPPING, + "medicalRecordId", TEXT_KEYWORD_MAPPING, + "name", TEXT_KEYWORD_MAPPING, + "contact", TEXT_KEYWORD_MAPPING ); + + private static final Map kitRequestShippingMigrate2 = Map.of ( + "phone", TEXT_KEYWORD_MAPPING, + "fax", TEXT_KEYWORD_MAPPING, + "faxSent", DATE_MAPPING, //DATE? + "faxSentBy", TEXT_KEYWORD_MAPPING, + "faxConfirmed", DATE_MAPPING, //DATE? + "faxSent2", DATE_MAPPING, //DATE? + "faxSent2By", TEXT_KEYWORD_MAPPING, + "faxConfirmed2", DATE_MAPPING, //DATE? + "faxSent3", DATE_MAPPING, //DATE? + "faxSent3By", TEXT_KEYWORD_MAPPING); + + private static final Map medicalRecordMapping3 = Map.of ( + "faxConfirmed3", DATE_MAPPING, //DATE? + "mrReceived", DATE_MAPPING, //DATE? + "followUps", TEXT_KEYWORD_MAPPING, + "mrDocument", TEXT_KEYWORD_MAPPING, + "mrDocumentFileName", TEXT_KEYWORD_MAPPING, + "mrProblem", BOOLEAN_MAPPING, + "mrProblemText", TEXT_KEYWORD_MAPPING, + "unableObtain", BOOLEAN_MAPPING, + "unableObtainText", TEXT_KEYWORD_MAPPING, + "duplicate", BOOLEAN_MAPPING); + + private static final Map medicalRecordMapping4 = Map.of ( + "followUpRequired", BOOLEAN_MAPPING, + "followUpRequiredText", TEXT_KEYWORD_MAPPING, + "international", BOOLEAN_MAPPING, + "crRequired", BOOLEAN_MAPPING, + "pathologyPresent", TEXT_KEYWORD_MAPPING, + "notes", TEXT_KEYWORD_MAPPING, + "additionalValuesJson", TEXT_KEYWORD_MAPPING, + "reviewMedicalRecord", TEXT_KEYWORD_MAPPING + ); + + +} From d5884754d31363b0fef9a802f59d0dfb824beed6 Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 4 Nov 2021 16:55:28 +0400 Subject: [PATCH 087/405] DDP-7117 | finish KitRequestShippingMigrate --- .../KitRequestShippingMigrate.java | 101 +++++++++++------- 1 file changed, 61 insertions(+), 40 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrate.java index 5a6f1e8e4..880a8655b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrate.java @@ -9,49 +9,70 @@ public class KitRequestShippingMigrate { private static final Map kitRequestShippingMigrate1 = Map.of ( + "kitTypeName", TEXT_KEYWORD_MAPPING, + "instanceName", TEXT_KEYWORD_MAPPING, + "bspCollaboratorParticipantId", TEXT_KEYWORD_MAPPING, + "bspCollaboratorSampleId", TEXT_KEYWORD_MAPPING, "ddpParticipantId", TEXT_KEYWORD_MAPPING, - "ddpInstanceId", TEXT_KEYWORD_MAPPING, - "institutionId", TEXT_KEYWORD_MAPPING, - "ddpInstitutionId", TEXT_KEYWORD_MAPPING, - "type", TEXT_KEYWORD_MAPPING, - "participantId", TEXT_KEYWORD_MAPPING, - "medicalRecordId", TEXT_KEYWORD_MAPPING, - "name", TEXT_KEYWORD_MAPPING, - "contact", TEXT_KEYWORD_MAPPING ); + "ddpLabel", TEXT_KEYWORD_MAPPING, + "dsmKitRequest", TEXT_KEYWORD_MAPPING, + "kitTypeId", TEXT_KEYWORD_MAPPING, + "externalOrderStatus", TEXT_KEYWORD_MAPPING, + "externalOrderNumber", TEXT_KEYWORD_MAPPING + ); private static final Map kitRequestShippingMigrate2 = Map.of ( - "phone", TEXT_KEYWORD_MAPPING, - "fax", TEXT_KEYWORD_MAPPING, - "faxSent", DATE_MAPPING, //DATE? - "faxSentBy", TEXT_KEYWORD_MAPPING, - "faxConfirmed", DATE_MAPPING, //DATE? - "faxSent2", DATE_MAPPING, //DATE? - "faxSent2By", TEXT_KEYWORD_MAPPING, - "faxConfirmed2", DATE_MAPPING, //DATE? - "faxSent3", DATE_MAPPING, //DATE? - "faxSent3By", TEXT_KEYWORD_MAPPING); - - private static final Map medicalRecordMapping3 = Map.of ( - "faxConfirmed3", DATE_MAPPING, //DATE? - "mrReceived", DATE_MAPPING, //DATE? - "followUps", TEXT_KEYWORD_MAPPING, - "mrDocument", TEXT_KEYWORD_MAPPING, - "mrDocumentFileName", TEXT_KEYWORD_MAPPING, - "mrProblem", BOOLEAN_MAPPING, - "mrProblemText", TEXT_KEYWORD_MAPPING, - "unableObtain", BOOLEAN_MAPPING, - "unableObtainText", TEXT_KEYWORD_MAPPING, - "duplicate", BOOLEAN_MAPPING); - - private static final Map medicalRecordMapping4 = Map.of ( - "followUpRequired", BOOLEAN_MAPPING, - "followUpRequiredText", TEXT_KEYWORD_MAPPING, - "international", BOOLEAN_MAPPING, - "crRequired", BOOLEAN_MAPPING, - "pathologyPresent", TEXT_KEYWORD_MAPPING, - "notes", TEXT_KEYWORD_MAPPING, - "additionalValuesJson", TEXT_KEYWORD_MAPPING, - "reviewMedicalRecord", TEXT_KEYWORD_MAPPING + "externalOrderDate", TEXT_KEYWORD_MAPPING, + "externalResponse", TEXT_KEYWORD_MAPPING, + "uploadReason", TEXT_KEYWORD_MAPPING, + "noReturn", BOOLEAN_MAPPING, + "createdBy", TEXT_KEYWORD_MAPPING, + "dsmKitRequestId", TEXT_KEYWORD_MAPPING, + "dsmKitId", TEXT_KEYWORD_MAPPING, + "kitComplete", BOOLEAN_MAPPING, + "labelUrlTo", TEXT_KEYWORD_MAPPING, + "labelUrlReturn", TEXT_KEYWORD_MAPPING + ); + + private static final Map kitRequestShippingMigrate3 = Map.of ( + "trackingToId", TEXT_KEYWORD_MAPPING, + "trackingReturnId", TEXT_KEYWORD_MAPPING, + "easypostTrackingToUrl", TEXT_KEYWORD_MAPPING, + "easypostTrackingReturnUrl", TEXT_KEYWORD_MAPPING, + "easypostToId", TEXT_KEYWORD_MAPPING, + "easypostShipmentStatus", BOOLEAN_MAPPING, + "scanDate", TEXT_KEYWORD_MAPPING, + "labelDate", TEXT_KEYWORD_MAPPING, + "error", BOOLEAN_MAPPING, + "message", TEXT_KEYWORD_MAPPING); + + private static final Map kitRequestShippingMigrate4 = Map.of ( + "receiveDate", TEXT_KEYWORD_MAPPING, + "deactivatedDate", TEXT_KEYWORD_MAPPING, + "easypostAddressIdTo", TEXT_KEYWORD_MAPPING, + "deactivationReason", TEXT_KEYWORD_MAPPING, + "trackingId", TEXT_KEYWORD_MAPPING, + "kitLabel", TEXT_KEYWORD_MAPPING, + "express", BOOLEAN_MAPPING, + "testResult", TEXT_KEYWORD_MAPPING, + "needsApproval", BOOLEAN_MAPPING, + "authorization", TEXT_KEYWORD_MAPPING + ); + + private static final Map kitRequestShippingMigrate5 = Map.of ( + "denialReason", TEXT_KEYWORD_MAPPING, + "authorizedBy", TEXT_KEYWORD_MAPPING, + "upsTrackingStatus", TEXT_KEYWORD_MAPPING, + "upsReturnStatus", TEXT_KEYWORD_MAPPING, + "CEOrder", BOOLEAN_MAPPING // ? what is this ? + ); + + private static final Map KitRequestShippingMerged = MapAdapter.of( + kitRequestShippingMigrate1, + kitRequestShippingMigrate2, + kitRequestShippingMigrate3, + kitRequestShippingMigrate4, + kitRequestShippingMigrate5 ); From 96711a31c700a92cf007a3868f8fcc2128258005 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 4 Nov 2021 18:02:46 +0400 Subject: [PATCH 088/405] DDP-7117 | add participant and participant data mappings --- .../ParticipantDataMigrate.java | 21 ++++++++ .../migrationscript/ParticipantMigrate.java | 51 +++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrate.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java new file mode 100644 index 000000000..1611b43e6 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java @@ -0,0 +1,21 @@ +package org.broadinstitute.dsm.model.elastic.migrationscript; + +import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.BOOLEAN_MAPPING; +import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.TEXT_KEYWORD_MAPPING; + +import java.util.Map; + +public class ParticipantDataMigrate { + + + private static final Map participantDataMigrate1 = Map.of ( + "participantDataId", TEXT_KEYWORD_MAPPING, + "ddpParticipantId", TEXT_KEYWORD_MAPPING, + "ddpInstanceId", TEXT_KEYWORD_MAPPING, + "fieldTypeId", TEXT_KEYWORD_MAPPING + ); + + + + +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrate.java new file mode 100644 index 000000000..34ed6b3fd --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrate.java @@ -0,0 +1,51 @@ +package org.broadinstitute.dsm.model.elastic.migrationscript; + +import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.BOOLEAN_MAPPING; +import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.DATE_MAPPING; +import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.TEXT_KEYWORD_MAPPING; + +import java.util.Map; + +public class ParticipantMigrate { + + + private static final Map participantMigrate1 = Map.of ( + "participantId", TEXT_KEYWORD_MAPPING, + "ddpParticipantId", TEXT_KEYWORD_MAPPING, + "assigneeIdMr", TEXT_KEYWORD_MAPPING, + "assigneeIdTissue", TEXT_KEYWORD_MAPPING, + "ddpInstanceId", TEXT_KEYWORD_MAPPING, + "instanceName", TEXT_KEYWORD_MAPPING, + "baseUrl", TEXT_KEYWORD_MAPPING, + "mrAttentionFlagD", TEXT_KEYWORD_MAPPING, + "tissueAttentionFlagD", TEXT_KEYWORD_MAPPING, + "auth0Token", BOOLEAN_MAPPING + ); + + private static final Map participantMigrate2 = Map.of ( + "notificationRecipients", TEXT_KEYWORD_MAPPING, + "migratedDdp", BOOLEAN_MAPPING, + "oncHistoryId", TEXT_KEYWORD_MAPPING, + "created", DATE_MAPPING, + "reviewed", DATE_MAPPING, + "crSent", DATE_MAPPING, //DATE? + "crReceived", DATE_MAPPING, //DATE? + "notes", TEXT_KEYWORD_MAPPING, + "minimalMr", TEXT_KEYWORD_MAPPING, + "abstractionReady", TEXT_KEYWORD_MAPPING + ); + + private static final Map participantMigrate3 = Map.of ( + "additionalValuesJson", TEXT_KEYWORD_MAPPING, + "exitDate", TEXT_KEYWORD_MAPPING, + "exitBy", TEXT_KEYWORD_MAPPING + ); + + private static final Map participantMerged = MapAdapter.of( + participantMigrate1, + participantMigrate2, + participantMigrate3 + ); + + +} From 32694cf969dc9bd55ca42cfc4f3c7e56c9028654 Mon Sep 17 00:00:00 2001 From: Nika Date: Fri, 5 Nov 2021 13:53:12 +0400 Subject: [PATCH 089/405] DDP-7117 | start writing migration script --- .../migrationscript/MedicalRecordMigrate.java | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java index bcc126910..7b83cd79b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java @@ -12,6 +12,9 @@ import org.broadinstitute.dsm.db.structure.ColumnName; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.index.IndexRequest; + import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.*; public class MedicalRecordMigrate { @@ -65,8 +68,28 @@ public class MedicalRecordMigrate { protected static final Map medicalRecordMappingMerged = MapAdapter.of(medicalRecordMapping1, medicalRecordMapping2, medicalRecordMapping3, - medicalRecordMapping4);; + medicalRecordMapping4); + + + public static void exportMedicalRecordsToES() { + + // pt id -> List + Map> medicalRecords = MedicalRecord.getMedicalRecords("brain"); + + BulkRequest request = new BulkRequest(); + for (Map.Entry> entry: medicalRecords.entrySet()) { + String participantId = entry.getKey(); + List medicalRecordList = entry.getValue(); + + IndexRequest indexRequest = new IndexRequest("participants_structured.cmi.cmi-brain"); + // dsm { properties { + Map + + indexRequest.source + } + + } protected List collectMedicalRecordColumns() { Class medicalRecordClass = MedicalRecord.class; From a03d72b24b2650ca07654da190184d940cbaee4b Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 5 Nov 2021 14:21:54 +0400 Subject: [PATCH 090/405] DDP-7117 | add method to transform medicalrecord object to map --- .../migrationscript/MedicalRecordMigrate.java | 23 ++++++++++++++++++- .../MedicalRecordMigrateTest.java | 8 +++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java index 7b83cd79b..e8bfb7bc6 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java @@ -12,6 +12,8 @@ import org.broadinstitute.dsm.db.structure.ColumnName; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.index.IndexRequest; @@ -83,7 +85,21 @@ public static void exportMedicalRecordsToES() { List medicalRecordList = entry.getValue(); IndexRequest indexRequest = new IndexRequest("participants_structured.cmi.cmi-brain"); - // dsm { properties { + ElasticSearch elasticSearch = new ElasticSearch(); + ElasticSearchParticipantDto participantById = + elasticSearch.getParticipantById("participants_structured.cmi.cmi-brain", participantId); + String participantGuid = participantById.getParticipantId(); + indexRequest.id(participantGuid); + List> transformedList = transformMedicalRecordToMap(medicalRecordList); + + /* + dsm { + medicalRecords { + id: 10, + medicalRecordId: 10 + } + } + */ Map indexRequest.source @@ -91,6 +107,11 @@ public static void exportMedicalRecordsToES() { } + static List> transformMedicalRecordToMap(List medicalRecordList) { + + return null; + } + protected List collectMedicalRecordColumns() { Class medicalRecordClass = MedicalRecord.class; Field[] fields = medicalRecordClass.getDeclaredFields(); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java index e04b292a9..77a3c60d2 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java @@ -6,6 +6,7 @@ import java.util.List; import java.util.Map; +import org.broadinstitute.dsm.db.MedicalRecord; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.statics.DBConstants; import org.junit.Test; @@ -33,4 +34,11 @@ public void medicalRecordMappingMerged() { MedicalRecordMigrate.medicalRecordMappingMerged; assertEquals(37, medicalRecordMappingMerged.size()); } + + @Test + public void transformMedicalRecordToMap() { + List medicalRecords = Arrays.asList(new MedicalRecord("1", "2", "3", "TYPE")); + List> listOfMaps = MedicalRecordMigrate.transformMedicalRecordToMap(medicalRecords); + + } } \ No newline at end of file From 62b68ea062809128aabbeb00bbf79a0b2d180ba8 Mon Sep 17 00:00:00 2001 From: Nika Date: Fri, 5 Nov 2021 14:53:25 +0400 Subject: [PATCH 091/405] DDP-7117 | add main logic to serialization of MedicalRecord to Map --- .../migrationscript/MedicalRecordMigrate.java | 36 ++++++++++++++----- .../MedicalRecordMigrateTest.java | 7 +++- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java index e8bfb7bc6..aec941bbd 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java @@ -99,26 +99,46 @@ public static void exportMedicalRecordsToES() { medicalRecordId: 10 } } - */ - Map - - indexRequest.source +// */ +// Map +// +// indexRequest.source } } static List> transformMedicalRecordToMap(List medicalRecordList) { - + List columnNames = collectMedicalRecordColumns(); + List> result = new ArrayList<>(); + for (MedicalRecord medicalRecord: medicalRecordList) { + Map map = new HashMap<>(); + Class aClass = medicalRecord.getClass(); + Field[] declaredFields = aClass.getDeclaredFields(); + for (Field declaredField : declaredFields) { + ColumnName annotation = declaredField.getAnnotation(ColumnName.class); + if (annotation != null) { + try { + declaredField.setAccessible(true); + Object o = declaredField.get(medicalRecord); + String key = Util.underscoresToCamelCase(annotation.value()); + map.put(key, o); + } catch (IllegalAccessException e) { + e.printStackTrace(); + } + } + } + result.add(map); + } return null; } - protected List collectMedicalRecordColumns() { + protected static List collectMedicalRecordColumns() { Class medicalRecordClass = MedicalRecord.class; Field[] fields = medicalRecordClass.getDeclaredFields(); List columnNames = Arrays.stream(fields) .map(AccessibleObject::getAnnotations) - .map(annotations -> Arrays.stream(annotations).filter(this::isColumnNameType).findFirst()) + .map(annotations -> Arrays.stream(annotations).filter(MedicalRecordMigrate::isColumnNameType).findFirst()) .filter(Optional::isPresent) .map(Optional::get) .map(annotation -> ((ColumnName) annotation).value()) @@ -132,7 +152,7 @@ protected List swapToCamelCases(List columnNames) { .collect(Collectors.toList()); } - private boolean isColumnNameType(Annotation annotation) { + private static boolean isColumnNameType(Annotation annotation) { return annotation instanceof ColumnName; } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java index 77a3c60d2..83e9a3c1e 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java @@ -9,6 +9,7 @@ import org.broadinstitute.dsm.db.MedicalRecord; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.statics.DBConstants; +import org.junit.Assert; import org.junit.Test; public class MedicalRecordMigrateTest { @@ -39,6 +40,10 @@ public void medicalRecordMappingMerged() { public void transformMedicalRecordToMap() { List medicalRecords = Arrays.asList(new MedicalRecord("1", "2", "3", "TYPE")); List> listOfMaps = MedicalRecordMigrate.transformMedicalRecordToMap(medicalRecords); - + Map stringObjectMap = listOfMaps.get(0); + Assert.assertEquals("1", stringObjectMap.get("medicalRecordId")); + Assert.assertEquals("2", stringObjectMap.get("institutionId")); + Assert.assertEquals("3", stringObjectMap.get("ddpInstitutionId")); + Assert.assertEquals("TYPE", stringObjectMap.get("type")); } } \ No newline at end of file From 158004104aee02e25e3b5a75a45841fe28d32ad1 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 5 Nov 2021 15:21:47 +0400 Subject: [PATCH 092/405] DDP-7117 | add test, clean code a little bit --- .../broadinstitute/dsm/db/MedicalRecord.java | 27 ++++++++++++ .../migrationscript/MedicalRecordMigrate.java | 43 ++++++++----------- .../MedicalRecordMigrateTest.java | 11 +++++ 3 files changed, 56 insertions(+), 25 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java index a2a291187..b418f3339 100644 --- a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java +++ b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java @@ -45,9 +45,36 @@ public class MedicalRecord { "LEFT JOIN ddp_medical_record as m on (m.institution_id = inst.institution_id) WHERE p.participant_id = ?"; public static final String SQL_ORDER_BY = " ORDER BY p.ddp_participant_id, inst.ddp_institution_id ASC"; + @TableName ( + name = DBConstants.DDP_MEDICAL_RECORD, + alias = DBConstants.DDP_MEDICAL_RECORD_ALIAS, + primaryKey = DBConstants.MEDICAL_RECORD_ID, + columnPrefix = "") + @ColumnName (DBConstants.MEDICAL_RECORD_ID) private final String medicalRecordId; + + @TableName ( + name = DBConstants.DDP_INSTITUTION, + alias = DBConstants.DDP_INSTITUTION_ALIAS, + primaryKey = DBConstants.INSTITUTION_ID, + columnPrefix = "") + @ColumnName (DBConstants.INSTITUTION_ID) private String institutionId; + + @TableName ( + name = DBConstants.DDP_INSTITUTION, + alias = DBConstants.DDP_INSTITUTION_ALIAS, + primaryKey = DBConstants.DDP_INSTITUTION_ID, + columnPrefix = "") + @ColumnName (DBConstants.DDP_INSTITUTION_ID) private String ddpInstitutionId; + + @TableName ( + name = DBConstants.DDP_PARTICIPANT, + alias = DBConstants.DDP_PARTICIPANT_ALIAS, + primaryKey = DBConstants.DDP_PARTICIPANT_ID, + columnPrefix = "") + @ColumnName (DBConstants.DDP_PARTICIPANT_ID) private String ddpParticipantId; @TableName ( diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java index aec941bbd..873cd4352 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java @@ -91,45 +91,38 @@ public static void exportMedicalRecordsToES() { String participantGuid = participantById.getParticipantId(); indexRequest.id(participantGuid); List> transformedList = transformMedicalRecordToMap(medicalRecordList); - - /* - dsm { - medicalRecords { - id: 10, - medicalRecordId: 10 - } - } -// */ -// Map -// -// indexRequest.source + indexRequest.source(generateSource(transformedList)); + request.add(indexRequest); } + } + public static Map generateSource(List> transformedList) { + return Map.of("dsm", Map.of("medicalRecords", transformedList)); } + static List> transformMedicalRecordToMap(List medicalRecordList) { - List columnNames = collectMedicalRecordColumns(); List> result = new ArrayList<>(); + Class aClass = MedicalRecord.class; + Field[] declaredFields = aClass.getDeclaredFields(); for (MedicalRecord medicalRecord: medicalRecordList) { Map map = new HashMap<>(); - Class aClass = medicalRecord.getClass(); - Field[] declaredFields = aClass.getDeclaredFields(); for (Field declaredField : declaredFields) { ColumnName annotation = declaredField.getAnnotation(ColumnName.class); - if (annotation != null) { - try { - declaredField.setAccessible(true); - Object o = declaredField.get(medicalRecord); - String key = Util.underscoresToCamelCase(annotation.value()); - map.put(key, o); - } catch (IllegalAccessException e) { - e.printStackTrace(); - } + if (annotation == null) continue; + try { + declaredField.setAccessible(true); + Object fieldValue = declaredField.get(medicalRecord); + if (Objects.isNull(fieldValue)) continue; + String key = Util.underscoresToCamelCase(annotation.value()); + map.put(key, fieldValue); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); } } result.add(map); } - return null; + return result; } protected static List collectMedicalRecordColumns() { diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java index 83e9a3c1e..83506363e 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java @@ -46,4 +46,15 @@ public void transformMedicalRecordToMap() { Assert.assertEquals("3", stringObjectMap.get("ddpInstitutionId")); Assert.assertEquals("TYPE", stringObjectMap.get("type")); } + + @Test + public void generateSource() { + List medicalRecords = Arrays.asList(new MedicalRecord("1", "2", "3", "TYPE")); + List> listOfMaps = MedicalRecordMigrate.transformMedicalRecordToMap(medicalRecords); + Map resultMap = MedicalRecordMigrate.generateSource(listOfMaps); + Map dsm = (Map)resultMap.get("dsm"); + List> medicalRecords1 = (List>) dsm.get("medicalRecords"); + Object medicalRecordsId = medicalRecords1.get(0).get("medicalRecordId"); + assertEquals("1", medicalRecordsId); + } } \ No newline at end of file From 446c8b5b58980eb072828b1283412f34847aea00 Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 8 Nov 2021 17:01:55 +0400 Subject: [PATCH 093/405] DDP-7117 | introduce BulkExportFacade.java based on Facade pattern --- .../dsm/model/elastic/Util.java | 35 +++++++- .../export/UpsertDataRequestPayload.java | 5 +- .../migrationscript/BulkExportFacade.java | 47 ++++++++++ .../migrationscript/MedicalRecordMigrate.java | 88 ++++++++++--------- .../model/elastic/search/ElasticSearch.java | 4 +- .../MedicalRecordMigrateTest.java | 22 ++++- 6 files changed, 148 insertions(+), 53 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BulkExportFacade.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 14358fa7a..b60ed4e14 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -1,11 +1,11 @@ package org.broadinstitute.dsm.model.elastic; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.lang.reflect.Field; +import java.util.*; import java.util.stream.Collectors; +import com.google.gson.Gson; +import org.broadinstitute.dsm.db.structure.ColumnName; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; import org.broadinstitute.dsm.statics.ESObjectConstants; @@ -25,6 +25,7 @@ public class Util { ); public static final int FIRST_ELEMENT_INDEX = 0; public static final String UNDERSCORE_SEPARATOR = "_"; + public static final String DOC = "_doc"; public static String getQueryTypeFromId(String id) { String type; @@ -57,6 +58,32 @@ public static String underscoresToCamelCase(String fieldName) { return String.join("", words); } + public static List> transformObjectCollectionToCollectionMap(List values, Class aClass) { + List> result = new ArrayList<>(); + Field[] declaredFields = aClass.getDeclaredFields(); + for (Object obj: values) { + Map map = new HashMap<>(); + for (Field declaredField : declaredFields) { + ColumnName annotation = declaredField.getAnnotation(ColumnName.class); + if (annotation == null) continue; + try { + declaredField.setAccessible(true); + Object fieldValue = declaredField.get(obj); + if (Objects.isNull(fieldValue)) continue; + String key = underscoresToCamelCase(annotation.value()); + if (key.equals("followUps")) { + fieldValue = new Gson().toJson(fieldValue); + } + map.put(key, fieldValue); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + } + result.add(map); + } + return result; + } + public static class Constants { public static final String PROFILE = "profile"; public static final String PROFILE_HRUID = PROFILE + ".hruid"; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/UpsertDataRequestPayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/UpsertDataRequestPayload.java index d01212d86..b7102f020 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/UpsertDataRequestPayload.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/UpsertDataRequestPayload.java @@ -1,13 +1,12 @@ package org.broadinstitute.dsm.model.elastic.export; +import org.broadinstitute.dsm.model.elastic.Util; import org.elasticsearch.action.update.UpdateRequest; import java.util.Map; public class UpsertDataRequestPayload { - private static final String DOC = "_doc"; - private String index; private String type; private String id; @@ -25,7 +24,7 @@ private UpsertDataRequestPayload(Builder builder) { public UpdateRequest getUpdateRequest(Map data) { return new UpdateRequest() .index(index) - .type(DOC) + .type(Util.DOC) .id(id) .doc(data) .docAsUpsert(docAsUpsert) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BulkExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BulkExportFacade.java new file mode 100644 index 000000000..1a31546c2 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BulkExportFacade.java @@ -0,0 +1,47 @@ +package org.broadinstitute.dsm.model.elastic.migrationscript; + +import org.broadinstitute.dsm.util.ElasticSearchUtil; +import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; + +import java.io.IOException; +import java.util.Map; + +import static org.broadinstitute.dsm.model.elastic.Util.DOC; + +public class BulkExportFacade { + + private String index; + private BulkRequest bulkRequest; + + public BulkExportFacade(String index) { + this.index = index; + this.bulkRequest = new BulkRequest(); + } + + public void addDataToRequest(Map mapToUpsert, String docId) { + bulkRequest.add(createRequest(mapToUpsert, docId)); + + + } + + private UpdateRequest createRequest(Map mapToUpsert, String docId) { + UpdateRequest updateRequest = new UpdateRequest(index, DOC, docId); + updateRequest.doc(mapToUpsert); + return updateRequest; + } + + public void executeBulkUpsert() { + RestHighLevelClient client = ElasticSearchUtil.getClientInstance(); + try { + client.bulk(bulkRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + + +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java index 873cd4352..21acda440 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java @@ -1,27 +1,34 @@ package org.broadinstitute.dsm.model.elastic.migrationscript; +import java.io.IOException; import java.lang.annotation.Annotation; import java.lang.reflect.AccessibleObject; import java.lang.reflect.Field; import java.util.*; -import java.util.function.Function; -import java.util.function.Predicate; import java.util.stream.Collectors; import org.broadinstitute.dsm.db.MedicalRecord; import org.broadinstitute.dsm.db.structure.ColumnName; import org.broadinstitute.dsm.model.elastic.Util; -import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; +import org.broadinstitute.dsm.util.ElasticSearchUtil; +import org.broadinstitute.dsm.util.ParticipantUtil; import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import spark.utils.StringUtils; import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.*; public class MedicalRecordMigrate { + public static final ElasticSearch elasticSearch = new ElasticSearch(); + private static final Map medicalRecordMapping1 = Map.of ( "ddpParticipantId", TEXT_KEYWORD_MAPPING, "ddpInstanceId", TEXT_KEYWORD_MAPPING, @@ -74,26 +81,51 @@ public class MedicalRecordMigrate { public static void exportMedicalRecordsToES() { + Map> medicalRecords = (Map) MedicalRecord.getMedicalRecords("brain"); + BulkExportFacade bulkExportFacade = new BulkExportFacade("participants_structured.cmi.cmi-brain"); + bulkExportFacade.addDataToRequest(); + BulkRequest request = buildAndFillBulkRequest(medicalRecords); + RestHighLevelClient client = ElasticSearchUtil.getClientInstance(); + try { + client.bulk(request, RequestOptions.DEFAULT); + } catch (IOException e) { + throw new RuntimeException(); + } + } - // pt id -> List - Map> medicalRecords = MedicalRecord.getMedicalRecords("brain"); + private static BulkRequest buildAndFillBulkRequest(Map> participantRecords) { + for (Map.Entry> entry: participantRecords.entrySet()) { + String participantId = entry.getKey(); + List medicalRecordList = entry.getValue(); + participantId = getParticipantGuid(participantId); + if (StringUtils.isBlank(participantId)) continue; + List> transformedList = Util.transformObjectCollectionToCollectionMap(medicalRecordList, MedicalRecord.class); + setPrimaryId(transformedList); + } - BulkRequest request = new BulkRequest(); + return null; + } - for (Map.Entry> entry: medicalRecords.entrySet()) { - String participantId = entry.getKey(); - List medicalRecordList = entry.getValue(); + private static UpdateRequest buildUpdateRequest(String participantId, List> transformedList) { + UpdateRequest updateRequest = new UpdateRequest("participants_structured.cmi.cmi-brain", "_doc", participantId); + updateRequest.docAsUpsert(true); + updateRequest.doc(generateSource(transformedList)); + return updateRequest; + } + + private static void setPrimaryId(List> transformedList) { + for(Map map: transformedList) { + map.put("id", map.get("medicalRecordId")); + } + } - IndexRequest indexRequest = new IndexRequest("participants_structured.cmi.cmi-brain"); - ElasticSearch elasticSearch = new ElasticSearch(); + private static String getParticipantGuid(String participantId) { + if (!(ParticipantUtil.isGuid(participantId))) { ElasticSearchParticipantDto participantById = elasticSearch.getParticipantById("participants_structured.cmi.cmi-brain", participantId); - String participantGuid = participantById.getParticipantId(); - indexRequest.id(participantGuid); - List> transformedList = transformMedicalRecordToMap(medicalRecordList); - indexRequest.source(generateSource(transformedList)); - request.add(indexRequest); + participantId = participantById.getParticipantId(); } + return participantId; } public static Map generateSource(List> transformedList) { @@ -101,30 +133,6 @@ public static Map generateSource(List> transformedList) { } - static List> transformMedicalRecordToMap(List medicalRecordList) { - List> result = new ArrayList<>(); - Class aClass = MedicalRecord.class; - Field[] declaredFields = aClass.getDeclaredFields(); - for (MedicalRecord medicalRecord: medicalRecordList) { - Map map = new HashMap<>(); - for (Field declaredField : declaredFields) { - ColumnName annotation = declaredField.getAnnotation(ColumnName.class); - if (annotation == null) continue; - try { - declaredField.setAccessible(true); - Object fieldValue = declaredField.get(medicalRecord); - if (Objects.isNull(fieldValue)) continue; - String key = Util.underscoresToCamelCase(annotation.value()); - map.put(key, fieldValue); - } catch (IllegalAccessException e) { - throw new RuntimeException(e); - } - } - result.add(map); - } - return result; - } - protected static List collectMedicalRecordColumns() { Class medicalRecordClass = MedicalRecord.class; Field[] fields = medicalRecordClass.getDeclaredFields(); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java index 4f10c7410..a1d2b0a82 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java @@ -207,9 +207,9 @@ public ElasticSearchParticipantDto getParticipantById(String esParticipantsIndex logger.info("Collecting ES data"); try { searchResponse = ElasticSearchUtil.getClientInstance().search(searchRequest, RequestOptions.DEFAULT); - sourceAsMap = searchResponse.getHits().getHits()[0].getSourceAsMap(); + sourceAsMap = searchResponse.getHits().getHits().length > 0 ? searchResponse.getHits().getHits()[0].getSourceAsMap() : Map.of(); } catch (Exception e) { - throw new RuntimeException("Couldn't get participant from ES for instance " + esParticipantsIndex + " by short id: " + participantId, e); + throw new RuntimeException("Couldn't get participant from ES for instance " + esParticipantsIndex + " by id: " + participantId, e); } return parseSourceMap(sourceAsMap).orElseThrow(); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java index 83506363e..0bebd2839 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java @@ -6,14 +6,22 @@ import java.util.List; import java.util.Map; +import org.broadinstitute.dsm.TestHelper; import org.broadinstitute.dsm.db.MedicalRecord; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.statics.DBConstants; import org.junit.Assert; +import org.junit.BeforeClass; import org.junit.Test; public class MedicalRecordMigrateTest { + + @BeforeClass + public static void setUp() { + TestHelper.setupDB(); + } + @Test public void collectMedicalRecordColumns() { MedicalRecordMigrate medicalRecordMigrate = new MedicalRecordMigrate(); @@ -38,8 +46,8 @@ public void medicalRecordMappingMerged() { @Test public void transformMedicalRecordToMap() { - List medicalRecords = Arrays.asList(new MedicalRecord("1", "2", "3", "TYPE")); - List> listOfMaps = MedicalRecordMigrate.transformMedicalRecordToMap(medicalRecords); + List medicalRecords = Arrays.asList(new MedicalRecord("1", "2", "3", "TYPE")); + List> listOfMaps = Util.transformObjectCollectionToCollectionMap(medicalRecords, MedicalRecord.class); Map stringObjectMap = listOfMaps.get(0); Assert.assertEquals("1", stringObjectMap.get("medicalRecordId")); Assert.assertEquals("2", stringObjectMap.get("institutionId")); @@ -49,12 +57,18 @@ public void transformMedicalRecordToMap() { @Test public void generateSource() { - List medicalRecords = Arrays.asList(new MedicalRecord("1", "2", "3", "TYPE")); - List> listOfMaps = MedicalRecordMigrate.transformMedicalRecordToMap(medicalRecords); + List medicalRecords = Arrays.asList(new MedicalRecord("1", "2", "3", "TYPE")); + List> listOfMaps = Util.transformObjectCollectionToCollectionMap(medicalRecords, MedicalRecord.class); Map resultMap = MedicalRecordMigrate.generateSource(listOfMaps); Map dsm = (Map)resultMap.get("dsm"); List> medicalRecords1 = (List>) dsm.get("medicalRecords"); Object medicalRecordsId = medicalRecords1.get(0).get("medicalRecordId"); assertEquals("1", medicalRecordsId); } + + @Test + public void exportMedicalRecordsToES() { + MedicalRecordMigrate.exportMedicalRecordsToES(); + } + } \ No newline at end of file From f95a272b9f37b52ed3b577a92c5c1b0510579ed0 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 8 Nov 2021 17:31:40 +0400 Subject: [PATCH 094/405] DDP-7117 | refactor Exportable --- .../model/elastic/export/BaseExporter.java | 11 ----- .../export/ElasticDataExportAdapter.java | 33 +++++++++++++ ....java => ElasticMappingExportAdapter.java} | 24 ++++------ .../model/elastic/export/ExportFacade.java | 9 ++-- .../dsm/model/elastic/export/Exportable.java | 5 +- .../migrationscript/MedicalRecordMigrate.java | 46 ++++++++----------- 6 files changed, 64 insertions(+), 64 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticDataExportAdapter.java rename src/main/java/org/broadinstitute/dsm/model/elastic/export/{ElasticExportAdapter.java => ElasticMappingExportAdapter.java} (55%) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java index 4b0e0e5c5..a2b4f65d9 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java @@ -6,16 +6,5 @@ import java.util.Objects; public abstract class BaseExporter implements Exportable { - protected RestHighLevelClient clientInstance = ElasticSearchUtil.getClientInstance(); - protected UpsertDataRequestPayload upsertDataRequestPayload; - protected UpsertMappingRequestPayload upsertMappingRequestPayload; - - public void setUpdateRequestPayload(UpsertDataRequestPayload upsertDataRequestPayload) { - this.upsertDataRequestPayload = Objects.requireNonNull(upsertDataRequestPayload); - } - - public void setUpsertMappingRequestPayload(UpsertMappingRequestPayload upsertMappingRequestPayload) { - this.upsertMappingRequestPayload = Objects.requireNonNull(upsertMappingRequestPayload); - } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticDataExportAdapter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticDataExportAdapter.java new file mode 100644 index 000000000..05bc914ed --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticDataExportAdapter.java @@ -0,0 +1,33 @@ +package org.broadinstitute.dsm.model.elastic.export; + +import java.io.IOException; +import java.util.Map; + +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.client.RequestOptions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ElasticDataExportAdapter extends BaseExporter { + + private static final Logger logger = LoggerFactory.getLogger(ElasticDataExportAdapter.class); + + private UpsertDataRequestPayload upsertDataRequestPayload; + + public ElasticDataExportAdapter(UpsertDataRequestPayload upsertDataRequestPayload) { + this.upsertDataRequestPayload = upsertDataRequestPayload; + } + + + @Override + public void export(Map data) { + logger.info("initialize exporting data to ES"); + UpdateRequest updateRequest = upsertDataRequestPayload.getUpdateRequest(data); + try { + clientInstance.update(updateRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + throw new RuntimeException("Error occurred while exporting data to ES", e); + } + logger.info("successfully exported data to ES"); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticExportAdapter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticMappingExportAdapter.java similarity index 55% rename from src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticExportAdapter.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticMappingExportAdapter.java index cd16a5920..97413747e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticExportAdapter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticMappingExportAdapter.java @@ -1,33 +1,25 @@ package org.broadinstitute.dsm.model.elastic.export; +import java.io.IOException; +import java.util.Map; -import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.indices.PutMappingRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.util.Map; +public class ElasticMappingExportAdapter extends BaseExporter { -public class ElasticExportAdapter extends BaseExporter { + private static final Logger logger = LoggerFactory.getLogger(ElasticMappingExportAdapter.class); - private static final Logger logger = LoggerFactory.getLogger(ElasticExportAdapter.class); + private UpsertMappingRequestPayload upsertMappingRequestPayload; - @Override - public void exportData(Map data) { - logger.info("initialize exporting data to ES"); - UpdateRequest updateRequest = upsertDataRequestPayload.getUpdateRequest(data); - try { - clientInstance.update(updateRequest, RequestOptions.DEFAULT); - } catch (IOException e) { - throw new RuntimeException("Error occurred while exporting data to ES", e); - } - logger.info("successfully exported data to ES"); + public ElasticMappingExportAdapter(UpsertMappingRequestPayload upsertMappingRequestPayload) { + this.upsertMappingRequestPayload = upsertMappingRequestPayload; } @Override - public void exportMapping(Map mapping) { + public void export(Map mapping) { logger.info("initialize exporting mapping to ES"); PutMappingRequest putMappingRequest = upsertMappingRequestPayload.getPutMappingRequest(); putMappingRequest.source(mapping); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 4c51f2924..39eaae34f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -34,7 +34,6 @@ public class ExportFacade { public ExportFacade(ExportFacadePayload exportFacadePayload) { this.exportFacadePayload = Objects.requireNonNull(exportFacadePayload); - exportable = new ElasticExportAdapter(); searchable = new ElasticSearch(); } @@ -47,8 +46,8 @@ private void upsertMapping() { generator = new MappingGenerator(new TypeParser(), exportFacadePayload.getGeneratorPayload()); Map mappingToUpsert = generator.generate(); UpsertMappingRequestPayload upsertMappingRequestPayload = new UpsertMappingRequestPayload(exportFacadePayload.getIndex()); - exportable.setUpsertMappingRequestPayload(upsertMappingRequestPayload); - exportable.exportMapping(mappingToUpsert); + exportable = new ElasticMappingExportAdapter(upsertMappingRequestPayload); + exportable.export(mappingToUpsert); } private ESDsm fetchData() { @@ -89,8 +88,8 @@ private void upsertData(Map elasticDataToExport) { .withRetryOnConflict(5) .build(); logger.info("Built upsert data request payload"); - exportable.setUpdateRequestPayload(upsertDataRequestPayload); - exportable.exportData(elasticDataToExport); + exportable = new ElasticDataExportAdapter(upsertDataRequestPayload); + exportable.export(elasticDataToExport); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java index 9a7ff81c3..74a1f300d 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java @@ -3,8 +3,5 @@ import java.util.Map; public interface Exportable { - - void exportData(Map data); - - void exportMapping(Map mapping); + void export(Map source); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java index 21acda440..1ba4fa7d5 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java @@ -1,6 +1,5 @@ package org.broadinstitute.dsm.model.elastic.migrationscript; -import java.io.IOException; import java.lang.annotation.Annotation; import java.lang.reflect.AccessibleObject; import java.lang.reflect.Field; @@ -10,21 +9,16 @@ import org.broadinstitute.dsm.db.MedicalRecord; import org.broadinstitute.dsm.db.structure.ColumnName; import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.model.elastic.export.Exportable; import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; -import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.broadinstitute.dsm.util.ParticipantUtil; -import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; -import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestHighLevelClient; import spark.utils.StringUtils; import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.*; -public class MedicalRecordMigrate { +public class MedicalRecordMigrate implements Exportable { public static final ElasticSearch elasticSearch = new ElasticSearch(); @@ -78,22 +72,25 @@ public class MedicalRecordMigrate { protected static final Map medicalRecordMappingMerged = MapAdapter.of(medicalRecordMapping1, medicalRecordMapping2, medicalRecordMapping3, medicalRecordMapping4); + private final BulkExportFacade bulkExportFacade; + public MedicalRecordMigrate(String index) { + bulkExportFacade = new BulkExportFacade(index); + } + + + @Override + public void export(Map source) { + + } - public static void exportMedicalRecordsToES() { + public void exportMedicalRecordsToES() { Map> medicalRecords = (Map) MedicalRecord.getMedicalRecords("brain"); - BulkExportFacade bulkExportFacade = new BulkExportFacade("participants_structured.cmi.cmi-brain"); - bulkExportFacade.addDataToRequest(); - BulkRequest request = buildAndFillBulkRequest(medicalRecords); - RestHighLevelClient client = ElasticSearchUtil.getClientInstance(); - try { - client.bulk(request, RequestOptions.DEFAULT); - } catch (IOException e) { - throw new RuntimeException(); - } + fillBulkRequest(medicalRecords); + bulkExportFacade.executeBulkUpsert(); } - private static BulkRequest buildAndFillBulkRequest(Map> participantRecords) { + private static void fillBulkRequest(Map> participantRecords) { for (Map.Entry> entry: participantRecords.entrySet()) { String participantId = entry.getKey(); List medicalRecordList = entry.getValue(); @@ -101,16 +98,8 @@ private static BulkRequest buildAndFillBulkRequest(Map> par if (StringUtils.isBlank(participantId)) continue; List> transformedList = Util.transformObjectCollectionToCollectionMap(medicalRecordList, MedicalRecord.class); setPrimaryId(transformedList); + bulkExportFacade.addDataToRequest(generateSource(transformedList), participantId); } - - return null; - } - - private static UpdateRequest buildUpdateRequest(String participantId, List> transformedList) { - UpdateRequest updateRequest = new UpdateRequest("participants_structured.cmi.cmi-brain", "_doc", participantId); - updateRequest.docAsUpsert(true); - updateRequest.doc(generateSource(transformedList)); - return updateRequest; } private static void setPrimaryId(List> transformedList) { @@ -156,4 +145,5 @@ protected List swapToCamelCases(List columnNames) { private static boolean isColumnNameType(Annotation annotation) { return annotation instanceof ColumnName; } + } From 2455147e1a0491cdad84cd59d61dba3101884dbb Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 8 Nov 2021 18:01:39 +0400 Subject: [PATCH 095/405] DDP-7117 | make MedicalRecordMigrate implement Generator and Exportable --- .../migrationscript/MedicalRecordMigrate.java | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java index 1ba4fa7d5..ad36393a7 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java @@ -10,15 +10,15 @@ import org.broadinstitute.dsm.db.structure.ColumnName; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.Exportable; +import org.broadinstitute.dsm.model.elastic.export.generate.Generator; import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.util.ParticipantUtil; -import org.elasticsearch.action.update.UpdateRequest; import spark.utils.StringUtils; import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.*; -public class MedicalRecordMigrate implements Exportable { +public class MedicalRecordMigrate implements Generator, Exportable { public static final ElasticSearch elasticSearch = new ElasticSearch(); @@ -72,25 +72,35 @@ public class MedicalRecordMigrate implements Exportable { protected static final Map medicalRecordMappingMerged = MapAdapter.of(medicalRecordMapping1, medicalRecordMapping2, medicalRecordMapping3, medicalRecordMapping4); + private final BulkExportFacade bulkExportFacade; + private final String realm; - public MedicalRecordMigrate(String index) { + public MedicalRecordMigrate(String index, String realm) { bulkExportFacade = new BulkExportFacade(index); + this.realm = realm; } + @Override + public Map generate() { + Map> medicalRecords = (Map) MedicalRecord.getMedicalRecords(realm); + Map>> medicalRecordsToMap = transformParticipantRecordsToMap(medicalRecords); + return (Map) medicalRecordsToMap; + } @Override public void export(Map source) { + // bulkExportFacade.addDataToRequest(generateSource(transformedList), participantId); + } public void exportMedicalRecordsToES() { - Map> medicalRecords = (Map) MedicalRecord.getMedicalRecords("brain"); - fillBulkRequest(medicalRecords); bulkExportFacade.executeBulkUpsert(); } - private static void fillBulkRequest(Map> participantRecords) { + private Map>> transformParticipantRecordsToMap(Map> participantRecords) { + Map>> transformedMap = new HashMap<>(); for (Map.Entry> entry: participantRecords.entrySet()) { String participantId = entry.getKey(); List medicalRecordList = entry.getValue(); @@ -98,8 +108,9 @@ private static void fillBulkRequest(Map> participantRecords if (StringUtils.isBlank(participantId)) continue; List> transformedList = Util.transformObjectCollectionToCollectionMap(medicalRecordList, MedicalRecord.class); setPrimaryId(transformedList); - bulkExportFacade.addDataToRequest(generateSource(transformedList), participantId); + transformedMap.put(participantId, transformedList); } + return transformedMap; } private static void setPrimaryId(List> transformedList) { @@ -146,4 +157,5 @@ private static boolean isColumnNameType(Annotation annotation) { return annotation instanceof ColumnName; } + } From 4d2d4c15be45d95f5084245cec7e5a9252a97e11 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 8 Nov 2021 18:44:41 +0400 Subject: [PATCH 096/405] DDP-7117 | add bulk exportable interface --- .../elastic/migrationscript/BaseMigrator.java | 56 +++++++++++ .../migrationscript/BulkExportable.java | 7 ++ .../migrationscript/MedicalRecordMigrate.java | 97 ++----------------- .../MedicalRecordMigrateTest.java | 2 +- 4 files changed, 71 insertions(+), 91 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BulkExportable.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java new file mode 100644 index 000000000..677ff2ebb --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java @@ -0,0 +1,56 @@ +package org.broadinstitute.dsm.model.elastic.migrationscript; + +import java.util.List; +import java.util.Map; + +import org.broadinstitute.dsm.db.MedicalRecord; +import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearchable; +import org.broadinstitute.dsm.util.ParticipantUtil; +import spark.utils.StringUtils; + +public abstract class BaseMigrator implements BulkExportable { + private final ElasticSearchable elasticSearch; + protected String OBJECT = ""; + protected final BulkExportFacade bulkExportFacade; + protected final String realm; + + public BaseMigrator(String index, String realm, ElasticSearchable elasticSearchable) { + this.elasticSearch = elasticSearchable; + bulkExportFacade = new BulkExportFacade(index); + this.realm = realm; + } + + private void setPrimaryId(List> transformedList) { + for(Map map: transformedList) { + map.put("id", map.get("medicalRecordId")); + } + } + + private String getParticipantGuid(String participantId) { + if (!(ParticipantUtil.isGuid(participantId))) { + ElasticSearchParticipantDto participantById = + elasticSearch.getParticipantById("participants_structured.cmi.cmi-brain", participantId); + participantId = participantById.getParticipantId(); + } + return participantId; + } + + public Map generateSource(List> transformedList) { + return Map.of("dsm", Map.of(OBJECT, transformedList)); + } + + protected void fillBulkRequestWithTransformedMap(Map> participantRecords) { + for (Map.Entry> entry: participantRecords.entrySet()) { + String participantId = entry.getKey(); + List medicalRecordList = entry.getValue(); + participantId = getParticipantGuid(participantId); + if (StringUtils.isBlank(participantId)) continue; + List> transformedList = Util.transformObjectCollectionToCollectionMap(medicalRecordList, MedicalRecord.class); + setPrimaryId(transformedList); + bulkExportFacade.addDataToRequest(generateSource(transformedList), participantId); + } + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BulkExportable.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BulkExportable.java new file mode 100644 index 000000000..89210191e --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BulkExportable.java @@ -0,0 +1,7 @@ +package org.broadinstitute.dsm.model.elastic.migrationscript; + +public interface BulkExportable { + + void export(); + +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java index ad36393a7..cbd5c8b0b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java @@ -1,28 +1,14 @@ package org.broadinstitute.dsm.model.elastic.migrationscript; -import java.lang.annotation.Annotation; -import java.lang.reflect.AccessibleObject; -import java.lang.reflect.Field; import java.util.*; -import java.util.stream.Collectors; import org.broadinstitute.dsm.db.MedicalRecord; -import org.broadinstitute.dsm.db.structure.ColumnName; -import org.broadinstitute.dsm.model.elastic.Util; -import org.broadinstitute.dsm.model.elastic.export.Exportable; -import org.broadinstitute.dsm.model.elastic.export.generate.Generator; -import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; -import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; -import org.broadinstitute.dsm.util.ParticipantUtil; -import spark.utils.StringUtils; import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.*; -public class MedicalRecordMigrate implements Generator, Exportable { +public class MedicalRecordMigrate extends BaseMigrator { - public static final ElasticSearch elasticSearch = new ElasticSearch(); - private static final Map medicalRecordMapping1 = Map.of ( "ddpParticipantId", TEXT_KEYWORD_MAPPING, "ddpInstanceId", TEXT_KEYWORD_MAPPING, @@ -73,89 +59,20 @@ public class MedicalRecordMigrate implements Generator, Exportable { medicalRecordMapping3, medicalRecordMapping4); - private final BulkExportFacade bulkExportFacade; - private final String realm; + private String OBJECT = "medicalRecords"; public MedicalRecordMigrate(String index, String realm) { - bulkExportFacade = new BulkExportFacade(index); - this.realm = realm; + super(index, realm); } - @Override - public Map generate() { - Map> medicalRecords = (Map) MedicalRecord.getMedicalRecords(realm); - Map>> medicalRecordsToMap = transformParticipantRecordsToMap(medicalRecords); - return (Map) medicalRecordsToMap; - } - @Override - public void export(Map source) { - - // bulkExportFacade.addDataToRequest(generateSource(transformedList), participantId); - } - - public void exportMedicalRecordsToES() { + @Override + public void export() { + Map> medicalRecords = (Map) MedicalRecord.getMedicalRecords(realm); + fillBulkRequestWithTransformedMap(medicalRecords); bulkExportFacade.executeBulkUpsert(); } - private Map>> transformParticipantRecordsToMap(Map> participantRecords) { - Map>> transformedMap = new HashMap<>(); - for (Map.Entry> entry: participantRecords.entrySet()) { - String participantId = entry.getKey(); - List medicalRecordList = entry.getValue(); - participantId = getParticipantGuid(participantId); - if (StringUtils.isBlank(participantId)) continue; - List> transformedList = Util.transformObjectCollectionToCollectionMap(medicalRecordList, MedicalRecord.class); - setPrimaryId(transformedList); - transformedMap.put(participantId, transformedList); - } - return transformedMap; - } - - private static void setPrimaryId(List> transformedList) { - for(Map map: transformedList) { - map.put("id", map.get("medicalRecordId")); - } - } - - private static String getParticipantGuid(String participantId) { - if (!(ParticipantUtil.isGuid(participantId))) { - ElasticSearchParticipantDto participantById = - elasticSearch.getParticipantById("participants_structured.cmi.cmi-brain", participantId); - participantId = participantById.getParticipantId(); - } - return participantId; - } - - public static Map generateSource(List> transformedList) { - return Map.of("dsm", Map.of("medicalRecords", transformedList)); - } - - - protected static List collectMedicalRecordColumns() { - Class medicalRecordClass = MedicalRecord.class; - Field[] fields = medicalRecordClass.getDeclaredFields(); - - List columnNames = Arrays.stream(fields) - .map(AccessibleObject::getAnnotations) - .map(annotations -> Arrays.stream(annotations).filter(MedicalRecordMigrate::isColumnNameType).findFirst()) - .filter(Optional::isPresent) - .map(Optional::get) - .map(annotation -> ((ColumnName) annotation).value()) - .collect(Collectors.toList()); - return columnNames; - } - - protected List swapToCamelCases(List columnNames) { - return columnNames.stream() - .map(Util::underscoresToCamelCase) - .collect(Collectors.toList()); - } - - private static boolean isColumnNameType(Annotation annotation) { - return annotation instanceof ColumnName; - } - } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java index 0bebd2839..4864ce0ac 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java @@ -59,7 +59,7 @@ public void transformMedicalRecordToMap() { public void generateSource() { List medicalRecords = Arrays.asList(new MedicalRecord("1", "2", "3", "TYPE")); List> listOfMaps = Util.transformObjectCollectionToCollectionMap(medicalRecords, MedicalRecord.class); - Map resultMap = MedicalRecordMigrate.generateSource(listOfMaps); + Map resultMap = BaseMigrator.generateSource(listOfMaps); Map dsm = (Map)resultMap.get("dsm"); List> medicalRecords1 = (List>) dsm.get("medicalRecords"); Object medicalRecordsId = medicalRecords1.get(0).get("medicalRecordId"); From eadbe5df15269da1b635f19556e5aff8d47bda3a Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 9 Nov 2021 10:45:41 +0400 Subject: [PATCH 097/405] DDP-7117 | change Exportable interface, remove BulkExportable and use Exportable instead, finish refactoring MedicalRecordMigrate --- .../dsm/model/elastic/Util.java | 1 + .../export/ElasticDataExportAdapter.java | 6 +- .../export/ElasticMappingExportAdapter.java | 6 +- .../model/elastic/export/ExportFacade.java | 8 +- .../dsm/model/elastic/export/Exportable.java | 4 +- .../export/generate/BaseGenerator.java | 1 - .../export/generate/MappingGenerator.java | 3 +- .../export/generate/SourceGenerator.java | 5 +- .../export/process/CollectionProcessor.java | 4 +- .../elastic/migrationscript/BaseMigrator.java | 32 +++++-- .../migrationscript/BulkExportable.java | 7 -- .../migrationscript/MedicalRecordMigrate.java | 67 +------------- .../OncHistoryDetailsMigrate.java | 87 +++---------------- .../export/generate/MappingGeneratorTest.java | 5 +- .../MedicalRecordMigrateTest.java | 31 ++----- 15 files changed, 67 insertions(+), 200 deletions(-) delete mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BulkExportable.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index b60ed4e14..bca60df2f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -26,6 +26,7 @@ public class Util { public static final int FIRST_ELEMENT_INDEX = 0; public static final String UNDERSCORE_SEPARATOR = "_"; public static final String DOC = "_doc"; + public static final String ID = "id"; public static String getQueryTypeFromId(String id) { String type; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticDataExportAdapter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticDataExportAdapter.java index 05bc914ed..5cf7b2c78 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticDataExportAdapter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticDataExportAdapter.java @@ -13,14 +13,16 @@ public class ElasticDataExportAdapter extends BaseExporter { private static final Logger logger = LoggerFactory.getLogger(ElasticDataExportAdapter.class); private UpsertDataRequestPayload upsertDataRequestPayload; + private Map data; - public ElasticDataExportAdapter(UpsertDataRequestPayload upsertDataRequestPayload) { + public ElasticDataExportAdapter(UpsertDataRequestPayload upsertDataRequestPayload, Map data) { this.upsertDataRequestPayload = upsertDataRequestPayload; + this.data = data; } @Override - public void export(Map data) { + public void export() { logger.info("initialize exporting data to ES"); UpdateRequest updateRequest = upsertDataRequestPayload.getUpdateRequest(data); try { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticMappingExportAdapter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticMappingExportAdapter.java index 97413747e..e47cf7001 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticMappingExportAdapter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticMappingExportAdapter.java @@ -13,13 +13,15 @@ public class ElasticMappingExportAdapter extends BaseExporter { private static final Logger logger = LoggerFactory.getLogger(ElasticMappingExportAdapter.class); private UpsertMappingRequestPayload upsertMappingRequestPayload; + private Map mapping; - public ElasticMappingExportAdapter(UpsertMappingRequestPayload upsertMappingRequestPayload) { + public ElasticMappingExportAdapter(UpsertMappingRequestPayload upsertMappingRequestPayload, Map mapping) { this.upsertMappingRequestPayload = upsertMappingRequestPayload; + this.mapping = mapping; } @Override - public void export(Map mapping) { + public void export() { logger.info("initialize exporting mapping to ES"); PutMappingRequest putMappingRequest = upsertMappingRequestPayload.getPutMappingRequest(); putMappingRequest.source(mapping); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 39eaae34f..1641d1e1f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -46,8 +46,8 @@ private void upsertMapping() { generator = new MappingGenerator(new TypeParser(), exportFacadePayload.getGeneratorPayload()); Map mappingToUpsert = generator.generate(); UpsertMappingRequestPayload upsertMappingRequestPayload = new UpsertMappingRequestPayload(exportFacadePayload.getIndex()); - exportable = new ElasticMappingExportAdapter(upsertMappingRequestPayload); - exportable.export(mappingToUpsert); + exportable = new ElasticMappingExportAdapter(upsertMappingRequestPayload, mappingToUpsert); + exportable.export(); } private ESDsm fetchData() { @@ -88,8 +88,8 @@ private void upsertData(Map elasticDataToExport) { .withRetryOnConflict(5) .build(); logger.info("Built upsert data request payload"); - exportable = new ElasticDataExportAdapter(upsertDataRequestPayload); - exportable.export(elasticDataToExport); + exportable = new ElasticDataExportAdapter(upsertDataRequestPayload, elasticDataToExport); + exportable.export(); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java index 74a1f300d..69466ad26 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java @@ -1,7 +1,5 @@ package org.broadinstitute.dsm.model.elastic.export; -import java.util.Map; - public interface Exportable { - void export(Map source); + void export(); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java index 2274e11ea..a34fa1613 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java @@ -18,7 +18,6 @@ public abstract class BaseGenerator implements Generator, Collector { public static final String DSM_OBJECT = "dsm"; public static final String PROPERTIES = "properties"; - public static final String ID = "id"; protected static final Gson GSON = new Gson(); protected final Parser parser; protected GeneratorPayload generatorPayload; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java index 26f98e24b..3efaedf13 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java @@ -1,5 +1,6 @@ package org.broadinstitute.dsm.model.elastic.export.generate; +import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,7 +55,7 @@ protected Object parseSingleElement() { @Override protected Map getElementWithId(Object type) { return Map.of( - ID, Map.of(TYPE, TYPE_KEYWORD), + Util.ID, Map.of(TYPE, TYPE_KEYWORD), getDBElement().getColumnName(), type ); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java index 6278fef9a..676769bc3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java @@ -1,5 +1,6 @@ package org.broadinstitute.dsm.model.elastic.export.generate; +import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,7 +56,7 @@ protected Object getElementWithId(Object element) { private List> buildCollectionWithId(Object element) { return List.of(Map.of(getDBElement().getColumnName(), element, - ID, generatorPayload.getRecordId())); + Util.ID, generatorPayload.getRecordId())); } @Override @@ -77,7 +78,7 @@ protected Object constructSingleElement() { protected Object constructCollection() { logger.info("Constructing nested data"); Map collectionMap = new HashMap<>(); - collectionMap.put(ID, generatorPayload.getRecordId()); + collectionMap.put(Util.ID, generatorPayload.getRecordId()); Map mapWithParsedObjects = parseJsonValuesToObject(); collectionMap.putAll(mapWithParsedObjects); return List.of(collectionMap); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java index 5e0aae69f..abd03bae5 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java @@ -10,7 +10,7 @@ import java.util.function.Predicate; import org.broadinstitute.dsm.model.elastic.ESDsm; -import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; +import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.generate.Collector; import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; import org.slf4j.Logger; @@ -75,7 +75,7 @@ private void addNewRecordTo(List> fetchedRecords) { } private boolean isExistingRecord(Map eachRecord) { - return (double) eachRecord.get(BaseGenerator.ID) == (double) generatorPayload.getRecordId(); + return (double) eachRecord.get(Util.ID) == (double) generatorPayload.getRecordId(); } private void updateExistingRecord(Map eachRecord) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java index 677ff2ebb..e1b544ce8 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java @@ -5,52 +5,66 @@ import org.broadinstitute.dsm.db.MedicalRecord; import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.model.elastic.export.Exportable; import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchable; +import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ParticipantUtil; import spark.utils.StringUtils; -public abstract class BaseMigrator implements BulkExportable { +public abstract class BaseMigrator implements Exportable { + private final ElasticSearchable elasticSearch; - protected String OBJECT = ""; + protected String object; protected final BulkExportFacade bulkExportFacade; protected final String realm; + protected final String index; - public BaseMigrator(String index, String realm, ElasticSearchable elasticSearchable) { - this.elasticSearch = elasticSearchable; + public BaseMigrator(String index, String realm, String object) { bulkExportFacade = new BulkExportFacade(index); + this.elasticSearch = new ElasticSearch(); this.realm = realm; + this.index = index; + this.object = object; } private void setPrimaryId(List> transformedList) { for(Map map: transformedList) { - map.put("id", map.get("medicalRecordId")); + map.put(Util.ID, map.get(object)); } } private String getParticipantGuid(String participantId) { if (!(ParticipantUtil.isGuid(participantId))) { ElasticSearchParticipantDto participantById = - elasticSearch.getParticipantById("participants_structured.cmi.cmi-brain", participantId); + elasticSearch.getParticipantById(index, participantId); participantId = participantById.getParticipantId(); } return participantId; } public Map generateSource(List> transformedList) { - return Map.of("dsm", Map.of(OBJECT, transformedList)); + return Map.of(ESObjectConstants.DSM, Map.of(object, transformedList)); } protected void fillBulkRequestWithTransformedMap(Map> participantRecords) { for (Map.Entry> entry: participantRecords.entrySet()) { String participantId = entry.getKey(); - List medicalRecordList = entry.getValue(); + List recordList = entry.getValue(); participantId = getParticipantGuid(participantId); if (StringUtils.isBlank(participantId)) continue; - List> transformedList = Util.transformObjectCollectionToCollectionMap(medicalRecordList, MedicalRecord.class); + List> transformedList = Util.transformObjectCollectionToCollectionMap(recordList, MedicalRecord.class); setPrimaryId(transformedList); bulkExportFacade.addDataToRequest(generateSource(transformedList), participantId); } } + + protected abstract Map> getDataByRealm(); + + @Override + public void export() { + fillBulkRequestWithTransformedMap(getDataByRealm()); + bulkExportFacade.executeBulkUpsert(); + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BulkExportable.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BulkExportable.java deleted file mode 100644 index 89210191e..000000000 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BulkExportable.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.broadinstitute.dsm.model.elastic.migrationscript; - -public interface BulkExportable { - - void export(); - -} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java index cbd5c8b0b..c53e87cca 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java @@ -3,76 +3,17 @@ import java.util.*; import org.broadinstitute.dsm.db.MedicalRecord; - -import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.*; +import org.broadinstitute.dsm.statics.ESObjectConstants; public class MedicalRecordMigrate extends BaseMigrator { - - private static final Map medicalRecordMapping1 = Map.of ( - "ddpParticipantId", TEXT_KEYWORD_MAPPING, - "ddpInstanceId", TEXT_KEYWORD_MAPPING, - "institutionId", TEXT_KEYWORD_MAPPING, - "ddpInstitutionId", TEXT_KEYWORD_MAPPING, - "type", TEXT_KEYWORD_MAPPING, - "participantId", TEXT_KEYWORD_MAPPING, - "medicalRecordId", TEXT_KEYWORD_MAPPING, - "name", TEXT_KEYWORD_MAPPING, - "contact", TEXT_KEYWORD_MAPPING ); - - private static final Map medicalRecordMapping2 = Map.of ( - "phone", TEXT_KEYWORD_MAPPING, - "fax", TEXT_KEYWORD_MAPPING, - "faxSent", DATE_MAPPING, //DATE? - "faxSentBy", TEXT_KEYWORD_MAPPING, - "faxConfirmed", DATE_MAPPING, //DATE? - "faxSent2", DATE_MAPPING, //DATE? - "faxSent2By", TEXT_KEYWORD_MAPPING, - "faxConfirmed2", DATE_MAPPING, //DATE? - "faxSent3", DATE_MAPPING, //DATE? - "faxSent3By", TEXT_KEYWORD_MAPPING); - - private static final Map medicalRecordMapping3 = Map.of ( - "faxConfirmed3", DATE_MAPPING, //DATE? - "mrReceived", DATE_MAPPING, //DATE? - "followUps", TEXT_KEYWORD_MAPPING, - "mrDocument", TEXT_KEYWORD_MAPPING, - "mrDocumentFileName", TEXT_KEYWORD_MAPPING, - "mrProblem", BOOLEAN_MAPPING, - "mrProblemText", TEXT_KEYWORD_MAPPING, - "unableObtain", BOOLEAN_MAPPING, - "unableObtainText", TEXT_KEYWORD_MAPPING, - "duplicate", BOOLEAN_MAPPING); - - private static final Map medicalRecordMapping4 = Map.of ( - "followUpRequired", BOOLEAN_MAPPING, - "followUpRequiredText", TEXT_KEYWORD_MAPPING, - "international", BOOLEAN_MAPPING, - "crRequired", BOOLEAN_MAPPING, - "pathologyPresent", TEXT_KEYWORD_MAPPING, - "notes", TEXT_KEYWORD_MAPPING, - "additionalValuesJson", TEXT_KEYWORD_MAPPING, - "reviewMedicalRecord", TEXT_KEYWORD_MAPPING - ); - - protected static final Map medicalRecordMappingMerged = MapAdapter.of(medicalRecordMapping1, medicalRecordMapping2, - medicalRecordMapping3, - medicalRecordMapping4); - - private String OBJECT = "medicalRecords"; - public MedicalRecordMigrate(String index, String realm) { - super(index, realm); + super(index, realm, ESObjectConstants.MEDICAL_RECORDS); } - - @Override - public void export() { - Map> medicalRecords = (Map) MedicalRecord.getMedicalRecords(realm); - fillBulkRequestWithTransformedMap(medicalRecords); - bulkExportFacade.executeBulkUpsert(); + protected Map> getDataByRealm() { + return (Map) MedicalRecord.getMedicalRecords(realm); } - } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java index 6dff29d81..63a2bb63e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java @@ -1,89 +1,24 @@ package org.broadinstitute.dsm.model.elastic.migrationscript; +import org.broadinstitute.dsm.statics.ESObjectConstants; + import java.util.HashMap; +import java.util.List; import java.util.Map; import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.*; import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.TEXT_KEYWORD_MAPPING; -public class OncHistoryDetailsMigrate { - - private static final Map oncHistoryDetails1 = Map.of ( - "ddpParticipantId", TEXT_KEYWORD_MAPPING, - "oncHistoryDetailId", TEXT_KEYWORD_MAPPING, - "request", TEXT_KEYWORD_MAPPING, - "deleted", BOOLEAN_MAPPING, - "faxSent", DATE_MAPPING, //DATE? - "tissueReceived", TEXT_KEYWORD_MAPPING, - "medicalRecordId", TEXT_KEYWORD_MAPPING, - "datePx", DATE_MAPPING, //DATE? - "typePx", TEXT_KEYWORD_MAPPING, - "locationPx", TEXT_KEYWORD_MAPPING ); - - private static final Map oncHistoryDetails2 = Map.of ( - "histology", TEXT_KEYWORD_MAPPING, - "accessionNumber", TEXT_KEYWORD_MAPPING, - "facility", TEXT_KEYWORD_MAPPING, - "phone", TEXT_KEYWORD_MAPPING, - "fax", TEXT_KEYWORD_MAPPING, - "notes", TEXT_KEYWORD_MAPPING, - "additionalValuesJson", TEXT_KEYWORD_MAPPING, - "faxSentBy", TEXT_KEYWORD_MAPPING, - "faxConfirmed", DATE_MAPPING, //DATE? - "faxSent2", DATE_MAPPING ); //DATE? - - private static final Map oncHistoryDetails3 = Map.of ( - "faxSent2By", TEXT_KEYWORD_MAPPING, - "faxConfirmed2", DATE_MAPPING, //DATE? - "faxSent3", DATE_MAPPING, - "faxSent3By", TEXT_KEYWORD_MAPPING, - "faxConfirmed3", DATE_MAPPING, - "tissueReceived", TEXT_KEYWORD_MAPPING, - "tissueProblemOption", TEXT_KEYWORD_MAPPING, - "gender", TEXT_KEYWORD_MAPPING, - "destructionPolicy", TEXT_KEYWORD_MAPPING, - "unableObtainTissue", BOOLEAN_MAPPING); - - private static final Map oncHistoryDetails4 = Map.of ( - "tissueId", TEXT_KEYWORD_MAPPING, - "countReceived", TEXT_KEYWORD_MAPPING, - "tissueType", TEXT_KEYWORD_MAPPING, - "tissueSite", TEXT_KEYWORD_MAPPING, - "hE", TEXT_KEYWORD_MAPPING, - "pathologyReport", TEXT_KEYWORD_MAPPING, - "collaboratorSampleId", TEXT_KEYWORD_MAPPING, - "blockSent", DATE_MAPPING, // DATE? - "scrollsReceived", DATE_MAPPING, // DATE? - "skId", TEXT_KEYWORD_MAPPING - ); +public class OncHistoryDetailsMigrate extends BaseMigrator { - private static final Map oncHistoryDetails5 = Map.of ( - "smId", TEXT_KEYWORD_MAPPING, - "sentGp", DATE_MAPPING, // DATE?, - "firstSmId", TEXT_KEYWORD_MAPPING, - "additionalTissueValueJson", TEXT_KEYWORD_MAPPING, - "expectedReturn", DATE_MAPPING, // DATE????????????? - "returnDate", DATE_MAPPING, // DATE? - "returnFedexId", TEXT_KEYWORD_MAPPING, - "shlWorkNumber", TEXT_KEYWORD_MAPPING, - "tumorPercentage", TEXT_KEYWORD_MAPPING, - "tissueSequence", TEXT_KEYWORD_MAPPING - ); - private static final Map oncHistoryDetails6 = Map.of ( - "scrollsCount", TEXT_KEYWORD_MAPPING, - "ussCount", TEXT_KEYWORD_MAPPING, - "hECount", TEXT_KEYWORD_MAPPING, - "blocksCount", TEXT_KEYWORD_MAPPING - ); + public OncHistoryDetailsMigrate(String index, String realm) { + super(index, realm, ESObjectConstants.ONC_HISTORY_DETAIL_RECORDS); + } - private static final Map oncHistoryDetailsMerged = MapAdapter.of( - oncHistoryDetails1, - oncHistoryDetails2, - oncHistoryDetails3, - oncHistoryDetails4, - oncHistoryDetails5, - oncHistoryDetails6 - ); + @Override + protected Map> getDataByRealm() { + return null; + } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java index 273537caa..02a2e49b2 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java @@ -3,6 +3,7 @@ import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; +import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import org.broadinstitute.dsm.model.elastic.export.TestPatchUtil; import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; @@ -81,7 +82,7 @@ public void getFieldWithTypeCollectionTrue() { TestMappingGenerator generator = TestMappingGenerator.of(generatorPayload); generator.getOuterPropertyByAlias().setIsCollection(true); Object fieldWithType = generator.getFieldWithElement(); - Assert.assertTrue(((Map) fieldWithType).containsKey(BaseGenerator.ID)); + Assert.assertTrue(((Map) fieldWithType).containsKey(Util.ID)); } @Test @@ -95,7 +96,7 @@ public void generateMapping() { Map dsmLevelProperty = Map.of(generator.getOuterPropertyByAlias().getPropertyName(), Map.of( MappingGenerator.TYPE, MappingGenerator.NESTED, MappingGenerator.PROPERTIES, Map.of(TestPatchUtil.MEDICAL_RECORD_COLUMN, Map.of(MappingGenerator.TYPE, "date"), - MappingGenerator.ID, Map.of(MappingGenerator.TYPE, MappingGenerator.TYPE_KEYWORD) + Util.ID, Map.of(MappingGenerator.TYPE, MappingGenerator.TYPE_KEYWORD) ))); Map dsmLevelProperties = Map.of(MappingGenerator.PROPERTIES, dsmLevelProperty); Map dsmLevel = Map.of(MappingGenerator.DSM_OBJECT, dsmLevelProperties); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java index 4864ce0ac..c00d9c938 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java @@ -16,34 +16,11 @@ public class MedicalRecordMigrateTest { - @BeforeClass public static void setUp() { TestHelper.setupDB(); } - @Test - public void collectMedicalRecordColumns() { - MedicalRecordMigrate medicalRecordMigrate = new MedicalRecordMigrate(); - List columnNames = medicalRecordMigrate.collectMedicalRecordColumns(); - assertTrue(columnNames.contains(DBConstants.MR_RECEIVED)); - } - - @Test - public void swapToCamelCases() { - MedicalRecordMigrate medicalRecordMigrate = new MedicalRecordMigrate(); - List columnNames = medicalRecordMigrate.collectMedicalRecordColumns(); - List camelCaseColumns = medicalRecordMigrate.swapToCamelCases(columnNames); - assertEquals("mrReceived", camelCaseColumns.stream().filter("mrReceived"::equals).findFirst().get()); - } - - @Test - public void medicalRecordMappingMerged() { - Map medicalRecordMappingMerged = - MedicalRecordMigrate.medicalRecordMappingMerged; - assertEquals(37, medicalRecordMappingMerged.size()); - } - @Test public void transformMedicalRecordToMap() { List medicalRecords = Arrays.asList(new MedicalRecord("1", "2", "3", "TYPE")); @@ -59,7 +36,8 @@ public void transformMedicalRecordToMap() { public void generateSource() { List medicalRecords = Arrays.asList(new MedicalRecord("1", "2", "3", "TYPE")); List> listOfMaps = Util.transformObjectCollectionToCollectionMap(medicalRecords, MedicalRecord.class); - Map resultMap = BaseMigrator.generateSource(listOfMaps); + MedicalRecordMigrate migrator = new MedicalRecordMigrate("", ""); + Map resultMap = migrator.generateSource(listOfMaps); Map dsm = (Map)resultMap.get("dsm"); List> medicalRecords1 = (List>) dsm.get("medicalRecords"); Object medicalRecordsId = medicalRecords1.get(0).get("medicalRecordId"); @@ -67,8 +45,9 @@ public void generateSource() { } @Test - public void exportMedicalRecordsToES() { - MedicalRecordMigrate.exportMedicalRecordsToES(); + public void export() { + BaseMigrator migrator = new MedicalRecordMigrate("participants_structured.cmi.cmi-brain","brain"); + migrator.export(); } } \ No newline at end of file From 73274454e4ed121089fdcab905e433d5dd58df06 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 9 Nov 2021 11:15:50 +0400 Subject: [PATCH 098/405] DDP-7117 | add onc history detail migrate class --- .../dsm/db/OncHistoryDetail.java | 3 +++ .../elastic/migrationscript/BaseMigrator.java | 9 +++++--- .../migrationscript/MedicalRecordMigrate.java | 2 +- .../OncHistoryDetailsMigrate.java | 5 +++-- .../OncHistoryDetailsMigrateTest.java | 21 +++++++++++++++++++ 5 files changed, 34 insertions(+), 6 deletions(-) create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrateTest.java diff --git a/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java b/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java index 19430bf46..48f6e4016 100644 --- a/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java +++ b/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java @@ -76,7 +76,10 @@ public class OncHistoryDetail { public static final String PROBLEM_OTHER = "other"; public static final String PROBLEM_OTHER_OLD = "Other"; + @ColumnName(DBConstants.ONC_HISTORY_DETAIL_ID) private String oncHistoryDetailId; + + @ColumnName(DBConstants.MEDICAL_RECORD_ID) private final String medicalRecordId; @ColumnName (DBConstants.DATE_PX) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java index e1b544ce8..0bd2fd495 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java @@ -18,11 +18,13 @@ public abstract class BaseMigrator implements Exportable { private final ElasticSearchable elasticSearch; protected String object; protected final BulkExportFacade bulkExportFacade; + private String primaryId; protected final String realm; protected final String index; - public BaseMigrator(String index, String realm, String object) { + public BaseMigrator(String index, String realm, String object, String primaryId) { bulkExportFacade = new BulkExportFacade(index); + this.primaryId = primaryId; this.elasticSearch = new ElasticSearch(); this.realm = realm; this.index = index; @@ -31,7 +33,7 @@ public BaseMigrator(String index, String realm, String object) { private void setPrimaryId(List> transformedList) { for(Map map: transformedList) { - map.put(Util.ID, map.get(object)); + map.put(Util.ID, map.get(primaryId)); } } @@ -54,9 +56,10 @@ protected void fillBulkRequestWithTransformedMap(Map> parti List recordList = entry.getValue(); participantId = getParticipantGuid(participantId); if (StringUtils.isBlank(participantId)) continue; - List> transformedList = Util.transformObjectCollectionToCollectionMap(recordList, MedicalRecord.class); + List> transformedList = Util.transformObjectCollectionToCollectionMap(recordList, this.getClass()); setPrimaryId(transformedList); bulkExportFacade.addDataToRequest(generateSource(transformedList), participantId); + System.err.println(participantId); //FOR TESTING } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java index c53e87cca..c948effc6 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java @@ -8,7 +8,7 @@ public class MedicalRecordMigrate extends BaseMigrator { public MedicalRecordMigrate(String index, String realm) { - super(index, realm, ESObjectConstants.MEDICAL_RECORDS); + super(index, realm, ESObjectConstants.MEDICAL_RECORDS, "medicalRecordId"); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java index 63a2bb63e..f31d0e319 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java @@ -1,5 +1,6 @@ package org.broadinstitute.dsm.model.elastic.migrationscript; +import org.broadinstitute.dsm.db.OncHistoryDetail; import org.broadinstitute.dsm.statics.ESObjectConstants; import java.util.HashMap; @@ -13,12 +14,12 @@ public class OncHistoryDetailsMigrate extends BaseMigrator { public OncHistoryDetailsMigrate(String index, String realm) { - super(index, realm, ESObjectConstants.ONC_HISTORY_DETAIL_RECORDS); + super(index, realm, ESObjectConstants.ONC_HISTORY_DETAIL_RECORDS, ""); } @Override protected Map> getDataByRealm() { - return null; + return (Map) OncHistoryDetail.getOncHistoryDetails(realm); } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrateTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrateTest.java new file mode 100644 index 000000000..32cb08664 --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrateTest.java @@ -0,0 +1,21 @@ +package org.broadinstitute.dsm.model.elastic.migrationscript; + +import static org.junit.Assert.*; + +import org.broadinstitute.dsm.TestHelper; +import org.junit.Before; +import org.junit.Test; + +public class OncHistoryDetailsMigrateTest { + + @Before + public void setUp() throws Exception { + TestHelper.setupDB(); + } + + @Test + public void export() { + OncHistoryDetailsMigrate oncHistoryDetailsMigrate = new OncHistoryDetailsMigrate("participants_structured.cmi.cmi-brain","brain"); + oncHistoryDetailsMigrate.export(); + } +} \ No newline at end of file From 944f8f55c71d1cdca103c0efa582d47716b9d681 Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 9 Nov 2021 11:44:59 +0400 Subject: [PATCH 099/405] DDP-7117 | initalize ParticipantMigrate --- .../dsm/model/elastic/Util.java | 40 ++++++------ .../elastic/migrationscript/BaseMigrator.java | 6 +- .../migrationscript/MedicalRecordMigrate.java | 2 +- .../OncHistoryDetailsMigrate.java | 5 +- .../migrationscript/ParticipantMigrate.java | 64 +++++++------------ 5 files changed, 51 insertions(+), 66 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index bca60df2f..61307d3e9 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -61,28 +61,32 @@ public static String underscoresToCamelCase(String fieldName) { public static List> transformObjectCollectionToCollectionMap(List values, Class aClass) { List> result = new ArrayList<>(); - Field[] declaredFields = aClass.getDeclaredFields(); for (Object obj: values) { - Map map = new HashMap<>(); - for (Field declaredField : declaredFields) { - ColumnName annotation = declaredField.getAnnotation(ColumnName.class); - if (annotation == null) continue; - try { - declaredField.setAccessible(true); - Object fieldValue = declaredField.get(obj); - if (Objects.isNull(fieldValue)) continue; - String key = underscoresToCamelCase(annotation.value()); - if (key.equals("followUps")) { - fieldValue = new Gson().toJson(fieldValue); - } - map.put(key, fieldValue); - } catch (IllegalAccessException e) { - throw new RuntimeException(e); + result.add(transformObjectToMap(aClass, obj)); + } + return result; + } + + public static Map transformObjectToMap(Class aClass, Object obj) { + Map map = new HashMap<>(); + Field[] declaredFields = aClass.getDeclaredFields(); + for (Field declaredField : declaredFields) { + ColumnName annotation = declaredField.getAnnotation(ColumnName.class); + if (annotation == null) continue; + try { + declaredField.setAccessible(true); + Object fieldValue = declaredField.get(obj); + if (Objects.isNull(fieldValue)) continue; + String key = underscoresToCamelCase(annotation.value()); + if (key.equals("followUps")) { + fieldValue = new Gson().toJson(fieldValue); } + map.put(key, fieldValue); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); } - result.add(map); } - return result; + return map; } public static class Constants { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java index 0bd2fd495..7b196d4cd 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java @@ -21,14 +21,16 @@ public abstract class BaseMigrator implements Exportable { private String primaryId; protected final String realm; protected final String index; + private Class aClass; - public BaseMigrator(String index, String realm, String object, String primaryId) { + public BaseMigrator(String index, String realm, String object, String primaryId, Class aClass) { bulkExportFacade = new BulkExportFacade(index); this.primaryId = primaryId; this.elasticSearch = new ElasticSearch(); this.realm = realm; this.index = index; this.object = object; + this.aClass = aClass; } private void setPrimaryId(List> transformedList) { @@ -56,7 +58,7 @@ protected void fillBulkRequestWithTransformedMap(Map> parti List recordList = entry.getValue(); participantId = getParticipantGuid(participantId); if (StringUtils.isBlank(participantId)) continue; - List> transformedList = Util.transformObjectCollectionToCollectionMap(recordList, this.getClass()); + List> transformedList = Util.transformObjectCollectionToCollectionMap(recordList, this.aClass); setPrimaryId(transformedList); bulkExportFacade.addDataToRequest(generateSource(transformedList), participantId); System.err.println(participantId); //FOR TESTING diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java index c948effc6..9a38a1c88 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java @@ -8,7 +8,7 @@ public class MedicalRecordMigrate extends BaseMigrator { public MedicalRecordMigrate(String index, String realm) { - super(index, realm, ESObjectConstants.MEDICAL_RECORDS, "medicalRecordId"); + super(index, realm, ESObjectConstants.MEDICAL_RECORDS, "medicalRecordId", MedicalRecord.class); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java index f31d0e319..aaa8360ec 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java @@ -3,18 +3,15 @@ import org.broadinstitute.dsm.db.OncHistoryDetail; import org.broadinstitute.dsm.statics.ESObjectConstants; -import java.util.HashMap; import java.util.List; import java.util.Map; -import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.*; -import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.TEXT_KEYWORD_MAPPING; public class OncHistoryDetailsMigrate extends BaseMigrator { public OncHistoryDetailsMigrate(String index, String realm) { - super(index, realm, ESObjectConstants.ONC_HISTORY_DETAIL_RECORDS, ""); + super(index, realm, ESObjectConstants.ONC_HISTORY_DETAIL_RECORDS, "oncHistoryDetailId", OncHistoryDetail.class); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrate.java index 34ed6b3fd..72c43aa24 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrate.java @@ -1,51 +1,33 @@ package org.broadinstitute.dsm.model.elastic.migrationscript; -import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.BOOLEAN_MAPPING; -import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.DATE_MAPPING; -import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.TEXT_KEYWORD_MAPPING; +import org.broadinstitute.dsm.db.Participant; +import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.model.elastic.export.Exportable; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearchable; import java.util.Map; -public class ParticipantMigrate { +public class ParticipantMigrate implements Exportable { + private String index; + private String realm; + private final ElasticSearchable elasticSearch; + protected final BulkExportFacade bulkExportFacade; + + public ParticipantMigrate(String index, String realm) { + this.index = index; + this.realm = realm; + this.bulkExportFacade = new BulkExportFacade(index); + this.elasticSearch = new ElasticSearch(); + } - private static final Map participantMigrate1 = Map.of ( - "participantId", TEXT_KEYWORD_MAPPING, - "ddpParticipantId", TEXT_KEYWORD_MAPPING, - "assigneeIdMr", TEXT_KEYWORD_MAPPING, - "assigneeIdTissue", TEXT_KEYWORD_MAPPING, - "ddpInstanceId", TEXT_KEYWORD_MAPPING, - "instanceName", TEXT_KEYWORD_MAPPING, - "baseUrl", TEXT_KEYWORD_MAPPING, - "mrAttentionFlagD", TEXT_KEYWORD_MAPPING, - "tissueAttentionFlagD", TEXT_KEYWORD_MAPPING, - "auth0Token", BOOLEAN_MAPPING - ); - - private static final Map participantMigrate2 = Map.of ( - "notificationRecipients", TEXT_KEYWORD_MAPPING, - "migratedDdp", BOOLEAN_MAPPING, - "oncHistoryId", TEXT_KEYWORD_MAPPING, - "created", DATE_MAPPING, - "reviewed", DATE_MAPPING, - "crSent", DATE_MAPPING, //DATE? - "crReceived", DATE_MAPPING, //DATE? - "notes", TEXT_KEYWORD_MAPPING, - "minimalMr", TEXT_KEYWORD_MAPPING, - "abstractionReady", TEXT_KEYWORD_MAPPING - ); - - private static final Map participantMigrate3 = Map.of ( - "additionalValuesJson", TEXT_KEYWORD_MAPPING, - "exitDate", TEXT_KEYWORD_MAPPING, - "exitBy", TEXT_KEYWORD_MAPPING - ); - - private static final Map participantMerged = MapAdapter.of( - participantMigrate1, - participantMigrate2, - participantMigrate3 - ); + @Override + public void export() { + Map participants = Participant.getParticipants(realm); + Map transformedObject = Util.transformObjectToMap(Participant.class, participants); + bulkExportFacade. + } } From 47a9bf1acbe2eeb3288c4f8060ef5be2c66a73fc Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 9 Nov 2021 15:36:58 +0400 Subject: [PATCH 100/405] DDP-7117 | add script to migrate participants info --- .../broadinstitute/dsm/db/Participant.java | 3 ++ .../dsm/model/elastic/export/Exportable.java | 14 +++++++++ .../elastic/migrationscript/BaseMigrator.java | 29 +++++++------------ .../migrationscript/ParticipantMigrate.java | 23 ++++++++++----- .../dsm/model/elastic/UtilTest.java | 20 +++++++++++++ .../MedicalRecordMigrateTest.java | 3 +- .../ParticipantMigrateTest.java | 21 ++++++++++++++ 7 files changed, 85 insertions(+), 28 deletions(-) create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrateTest.java diff --git a/src/main/java/org/broadinstitute/dsm/db/Participant.java b/src/main/java/org/broadinstitute/dsm/db/Participant.java index 3445fe99f..2c30de9a5 100644 --- a/src/main/java/org/broadinstitute/dsm/db/Participant.java +++ b/src/main/java/org/broadinstitute/dsm/db/Participant.java @@ -34,7 +34,10 @@ public class Participant { "LEFT JOIN ddp_participant_exit ex on (p.ddp_participant_id = ex.ddp_participant_id AND p.ddp_instance_id = ex.ddp_instance_id) " + "WHERE realm.instance_name = ? "; + @ColumnName(DBConstants.PARTICIPANT_ID) private final String participantId; + + @ColumnName(DBConstants.DDP_PARTICIPANT_ID) private final String ddpParticipantId; @TableName ( diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java index 69466ad26..f0e6a6d28 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java @@ -1,5 +1,19 @@ package org.broadinstitute.dsm.model.elastic.export; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; +import org.broadinstitute.dsm.util.ParticipantUtil; + public interface Exportable { void export(); + + default String getParticipantGuid(String participantId, String index) { + if (!(ParticipantUtil.isGuid(participantId))) { + ElasticSearchParticipantDto participantById = + new ElasticSearch().getParticipantById(index, participantId); + participantId = participantById.getParticipantId(); + } + return participantId; + } + } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java index 7b196d4cd..11d399c6e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java @@ -3,17 +3,15 @@ import java.util.List; import java.util.Map; -import org.broadinstitute.dsm.db.MedicalRecord; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.Exportable; +import org.broadinstitute.dsm.model.elastic.export.generate.Generator; import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; -import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchable; import org.broadinstitute.dsm.statics.ESObjectConstants; -import org.broadinstitute.dsm.util.ParticipantUtil; import spark.utils.StringUtils; -public abstract class BaseMigrator implements Exportable { +public abstract class BaseMigrator implements Exportable, Generator { private final ElasticSearchable elasticSearch; protected String object; @@ -22,6 +20,7 @@ public abstract class BaseMigrator implements Exportable { protected final String realm; protected final String index; private Class aClass; + private List> transformedList; public BaseMigrator(String index, String realm, String object, String primaryId, Class aClass) { bulkExportFacade = new BulkExportFacade(index); @@ -33,22 +32,14 @@ public BaseMigrator(String index, String realm, String object, String primaryId, this.aClass = aClass; } - private void setPrimaryId(List> transformedList) { + private void setPrimaryId() { for(Map map: transformedList) { map.put(Util.ID, map.get(primaryId)); } } - private String getParticipantGuid(String participantId) { - if (!(ParticipantUtil.isGuid(participantId))) { - ElasticSearchParticipantDto participantById = - elasticSearch.getParticipantById(index, participantId); - participantId = participantById.getParticipantId(); - } - return participantId; - } - - public Map generateSource(List> transformedList) { + @Override + public Map generate() { return Map.of(ESObjectConstants.DSM, Map.of(object, transformedList)); } @@ -56,11 +47,11 @@ protected void fillBulkRequestWithTransformedMap(Map> parti for (Map.Entry> entry: participantRecords.entrySet()) { String participantId = entry.getKey(); List recordList = entry.getValue(); - participantId = getParticipantGuid(participantId); + participantId = getParticipantGuid(participantId, index); if (StringUtils.isBlank(participantId)) continue; - List> transformedList = Util.transformObjectCollectionToCollectionMap(recordList, this.aClass); - setPrimaryId(transformedList); - bulkExportFacade.addDataToRequest(generateSource(transformedList), participantId); + transformedList = Util.transformObjectCollectionToCollectionMap(recordList, this.aClass); + setPrimaryId(); + bulkExportFacade.addDataToRequest(generate(), participantId); System.err.println(participantId); //FOR TESTING } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrate.java index 72c43aa24..6623ed8f8 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrate.java @@ -3,31 +3,40 @@ import org.broadinstitute.dsm.db.Participant; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.Exportable; +import org.broadinstitute.dsm.model.elastic.export.generate.Generator; import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchable; +import org.broadinstitute.dsm.statics.ESObjectConstants; import java.util.Map; -public class ParticipantMigrate implements Exportable { +public class ParticipantMigrate implements Exportable, Generator { private String index; private String realm; - private final ElasticSearchable elasticSearch; protected final BulkExportFacade bulkExportFacade; - + private Map transformedObject; + public ParticipantMigrate(String index, String realm) { this.index = index; this.realm = realm; this.bulkExportFacade = new BulkExportFacade(index); - this.elasticSearch = new ElasticSearch(); } @Override public void export() { Map participants = Participant.getParticipants(realm); - Map transformedObject = Util.transformObjectToMap(Participant.class, participants); - bulkExportFacade. - + for (Map.Entry entry: participants.entrySet()) { + Participant participant = entry.getValue(); + String participantId = entry.getKey(); + transformedObject = Util.transformObjectToMap(Participant.class, participant); + bulkExportFacade.addDataToRequest(generate(), getParticipantGuid(participantId, index)); + } + bulkExportFacade.executeBulkUpsert(); + } + @Override + public Map generate() { + return Map.of(ESObjectConstants.DSM, Map.of("participant", transformedObject)); } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java index ef609f135..432412d9b 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java @@ -2,6 +2,9 @@ import static org.junit.Assert.*; +import java.util.Map; + +import org.broadinstitute.dsm.db.Participant; import org.junit.Test; public class UtilTest { @@ -21,4 +24,21 @@ public void underscoresToCamelCase() { assertEquals("column", transformed3); assertEquals("column", transformed4); } + + @Test + public void transformObjectToMap() { + Participant participant = new Participant( + "1", "QWERTY", "assigneeMr", + "assigneeTissue", "instance", "2020-10-28", + "2020-10-28", "2020-10-28", "2020-10-28", + "ptNotes", true, true, + "additionalValuesJson", 1934283746283L); + Map transformedObject = Util.transformObjectToMap(Participant.class, participant); + assertEquals("1", transformedObject.get("participantId")); + assertEquals("QWERTY", transformedObject.get("ddpParticipantId")); + assertEquals("2020-10-28", transformedObject.get("created")); + assertEquals(true, transformedObject.get("minimalMr")); + assertEquals(1934283746283L, transformedObject.get("exitDate")); + } + } \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java index c00d9c938..3034384bd 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java @@ -9,7 +9,6 @@ import org.broadinstitute.dsm.TestHelper; import org.broadinstitute.dsm.db.MedicalRecord; import org.broadinstitute.dsm.model.elastic.Util; -import org.broadinstitute.dsm.statics.DBConstants; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -37,7 +36,7 @@ public void generateSource() { List medicalRecords = Arrays.asList(new MedicalRecord("1", "2", "3", "TYPE")); List> listOfMaps = Util.transformObjectCollectionToCollectionMap(medicalRecords, MedicalRecord.class); MedicalRecordMigrate migrator = new MedicalRecordMigrate("", ""); - Map resultMap = migrator.generateSource(listOfMaps); + Map resultMap = migrator.generate(); Map dsm = (Map)resultMap.get("dsm"); List> medicalRecords1 = (List>) dsm.get("medicalRecords"); Object medicalRecordsId = medicalRecords1.get(0).get("medicalRecordId"); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrateTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrateTest.java new file mode 100644 index 000000000..ee0ca725a --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrateTest.java @@ -0,0 +1,21 @@ +package org.broadinstitute.dsm.model.elastic.migrationscript; + +import static org.junit.Assert.*; + +import org.broadinstitute.dsm.TestHelper; +import org.junit.Before; +import org.junit.Test; + +public class ParticipantMigrateTest { + + @Before + public void setUp() { + TestHelper.setupDB(); + } + + @Test + public void export() { + ParticipantMigrate participantMigrate = new ParticipantMigrate("participants_structured.cmi.cmi-brain", "brain"); + participantMigrate.export(); + } +} \ No newline at end of file From b5a9e25e6b5cda8603208b97f8bf8020d93bac26 Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 9 Nov 2021 16:37:52 +0400 Subject: [PATCH 101/405] DDP-7117 | modifi classes related to ParticipantData in order to make ParticipantDataMigrate work --- .../ddp/participant/ParticipantDataDao.java | 54 ++++++++++++++++--- .../ddp/participant/ParticipantDataDto.java | 15 ++++++ .../ParticipantDataMigrate.java | 26 ++++----- 3 files changed, 75 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/dao/ddp/participant/ParticipantDataDao.java b/src/main/java/org/broadinstitute/dsm/db/dao/ddp/participant/ParticipantDataDao.java index 000b79b94..21363100a 100644 --- a/src/main/java/org/broadinstitute/dsm/db/dao/ddp/participant/ParticipantDataDao.java +++ b/src/main/java/org/broadinstitute/dsm/db/dao/ddp/participant/ParticipantDataDao.java @@ -61,18 +61,23 @@ public class ParticipantDataDao implements Dao { "last_changed = ?," + "changed_by = ?"; + public static final String SQL_SELECT_PARTICIPANTS_BY_REALM = "SELECT d.participant_data_id, d.ddp_participant_id, d.field_type_id, d.data " + + "FROM ddp_participant_data d " + + "LEFT JOIN ddp_instance realm on (d.ddp_instance_id = realm.ddp_instance_id) " + + "WHERE realm.instance_name = ? "; + private static final String SQL_UPDATE_DATA_TO_PARTICIPANT_DATA = "UPDATE ddp_participant_data SET data = ?, " + "last_changed = ?, changed_by = ? WHERE participant_data_id = ?"; private static final String SQL_GET_PARTICIPANT_DATA_BY_PARTICIPANT_IDS = SQL_ALL_PARTICIPANT_DATA + "WHERE ddp_participant_id IN (?)"; - private static final String PARTICIPANT_DATA_ID = "participant_data_id"; - private static final String DDP_PARTICIPANT_ID = "ddp_participant_id"; - private static final String DDP_INSTANCE_ID = "ddp_instance_id"; - private static final String FIELD_TYPE_ID = "field_type_id"; - private static final String DATA = "data"; - private static final String LAST_CHANGED = "last_changed"; - private static final String CHANGED_BY = "changed_by"; + public static final String PARTICIPANT_DATA_ID = "participant_data_id"; + public static final String DDP_PARTICIPANT_ID = "ddp_participant_id"; + public static final String DDP_INSTANCE_ID = "ddp_instance_id"; + public static final String FIELD_TYPE_ID = "field_type_id"; + public static final String DATA = "data"; + public static final String LAST_CHANGED = "last_changed"; + public static final String CHANGED_BY = "changed_by"; @Override @@ -253,7 +258,42 @@ public Map> getParticipantDataByParticipantIds( return participantDatasByParticipantIds; } + public Map> getParticipantDataByRealm(String realm) { + Map> participantDataByRealm = new HashMap<>(); + SimpleResult results = inTransaction((conn) -> { + SimpleResult execResult = new SimpleResult(); + try (PreparedStatement stmt = conn.prepareStatement(SQL_SELECT_PARTICIPANTS_BY_REALM)) { + stmt.setString(1, realm); + try(ResultSet rs = stmt.executeQuery()) { + while (rs.next()) { + ArrayList value = new ArrayList<>( + List.of(new ParticipantDataDto.Builder() + .withParticipantDataId(rs.getInt(PARTICIPANT_DATA_ID)) + .withDdpParticipantId(rs.getString(DDP_PARTICIPANT_ID)) + .withDdpInstanceId(rs.getInt(DDP_INSTANCE_ID)) + .withFieldTypeId(rs.getString(FIELD_TYPE_ID)) + .withData(rs.getString(DATA)) + .build() + ) + ); + participantDataByRealm.merge(rs.getString(DDP_PARTICIPANT_ID), value, (preValue, currentValue) -> { + preValue.addAll(currentValue); + return preValue; + }); + } + } + } + catch (SQLException ex) { + execResult.resultException = ex; + } + return execResult; + }); + if (results.resultException != null) { + throw new RuntimeException("Error getting participants data with ", results.resultException); + } + return participantDataByRealm; + } public Map> getParticipantDataByInstanceIdAndFilterQuery(int ddpInstanceId, String filterQuery) { Map> participantDatasByParticipantIds = new HashMap<>(); diff --git a/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java b/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java index 5b72fd729..7cebbd9fd 100644 --- a/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java +++ b/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java @@ -9,18 +9,33 @@ import com.google.gson.reflect.TypeToken; import lombok.Setter; import org.apache.commons.lang3.StringUtils; +import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; +import org.broadinstitute.dsm.db.structure.ColumnName; @Setter public class ParticipantDataDto { private static final Gson gson = new Gson(); + @ColumnName(ParticipantDataDao.PARTICIPANT_DATA_ID) private int participantDataId; + + @ColumnName(ParticipantDataDao.DDP_PARTICIPANT_ID) private String ddpParticipantId; + + @ColumnName(ParticipantDataDao.DDP_INSTANCE_ID) private int ddpInstanceId; + + @ColumnName(ParticipantDataDao.FIELD_TYPE_ID) private String fieldTypeId; + + @ColumnName(ParticipantDataDao.DATA) private String data; + + @ColumnName(ParticipantDataDao.LAST_CHANGED) private long lastChanged; + + @ColumnName(ParticipantDataDao.CHANGED_BY) private String changedBy; // We cache the json data map to avoid deserializing it multiple times. diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java index 1611b43e6..3e9bd0191 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java @@ -1,21 +1,21 @@ package org.broadinstitute.dsm.model.elastic.migrationscript; -import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.BOOLEAN_MAPPING; -import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.TEXT_KEYWORD_MAPPING; - -import java.util.Map; - -public class ParticipantDataMigrate { +import org.broadinstitute.dsm.db.OncHistoryDetail; +import org.broadinstitute.dsm.db.ParticipantData; +import org.broadinstitute.dsm.statics.ESObjectConstants; - private static final Map participantDataMigrate1 = Map.of ( - "participantDataId", TEXT_KEYWORD_MAPPING, - "ddpParticipantId", TEXT_KEYWORD_MAPPING, - "ddpInstanceId", TEXT_KEYWORD_MAPPING, - "fieldTypeId", TEXT_KEYWORD_MAPPING - ); - +import java.util.List; +import java.util.Map; +public class ParticipantDataMigrate extends BaseMigrator { + public ParticipantDataMigrate(String index, String realm) { + super(index, realm, ESObjectConstants.PARTICIPANT_DATA, "participantDataId", ParticipantData.class); + } + @Override + protected Map> getDataByRealm() { + return null; + } } From cb7d4178c9dbbe9ab3b9bd85c7e49914b63f1254 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 9 Nov 2021 17:12:51 +0400 Subject: [PATCH 102/405] create special case for dynamic fields --- .../ddp/participant/ParticipantDataDto.java | 3 -- .../dsm/model/elastic/Util.java | 54 +++++++++++++++++-- .../ParticipantDataMigrate.java | 13 +++-- 3 files changed, 57 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java b/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java index 7cebbd9fd..2fc4fae5d 100644 --- a/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java +++ b/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java @@ -29,13 +29,10 @@ public class ParticipantDataDto { @ColumnName(ParticipantDataDao.FIELD_TYPE_ID) private String fieldTypeId; - @ColumnName(ParticipantDataDao.DATA) private String data; - @ColumnName(ParticipantDataDao.LAST_CHANGED) private long lastChanged; - @ColumnName(ParticipantDataDao.CHANGED_BY) private String changedBy; // We cache the json data map to avoid deserializing it multiple times. diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 61307d3e9..d8bbfc711 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -1,10 +1,13 @@ package org.broadinstitute.dsm.model.elastic; +import java.io.IOException; import java.lang.reflect.Field; import java.util.*; import java.util.stream.Collectors; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; +import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.structure.ColumnName; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; @@ -22,7 +25,7 @@ public class Util { "r", new BaseGenerator.PropertyInfo(ESObjectConstants.PARTICIPANT_RECORD, false), "p", new BaseGenerator.PropertyInfo(ESObjectConstants.PARTICIPANT, false), "o", new BaseGenerator.PropertyInfo(ESObjectConstants.ONC_HISTORY, false) - ); + ); public static final int FIRST_ELEMENT_INDEX = 0; public static final String UNDERSCORE_SEPARATOR = "_"; public static final String DOC = "_doc"; @@ -32,7 +35,7 @@ public static String getQueryTypeFromId(String id) { String type; if (ParticipantUtil.isHruid(id)) { type = Constants.PROFILE_HRUID; - } else if (ParticipantUtil.isGuid(id)){ + } else if (ParticipantUtil.isGuid(id)) { type = Constants.PROFILE_GUID; } else if (ParticipantUtil.isLegacyAltPid(id)) { type = Constants.PROFILE_LEGACYALTPID; @@ -61,7 +64,7 @@ public static String underscoresToCamelCase(String fieldName) { public static List> transformObjectCollectionToCollectionMap(List values, Class aClass) { List> result = new ArrayList<>(); - for (Object obj: values) { + for (Object obj : values) { result.add(transformObjectToMap(aClass, obj)); } return result; @@ -72,11 +75,16 @@ public static Map transformObjectToMap(Class aClass, Object obj) Field[] declaredFields = aClass.getDeclaredFields(); for (Field declaredField : declaredFields) { ColumnName annotation = declaredField.getAnnotation(ColumnName.class); - if (annotation == null) continue; + if (annotation == null) { + continue; + } try { declaredField.setAccessible(true); Object fieldValue = declaredField.get(obj); - if (Objects.isNull(fieldValue)) continue; + if (Objects.isNull(fieldValue)) { + continue; + } + dynamicFieldsSpecialCase(fieldValue); String key = underscoresToCamelCase(annotation.value()); if (key.equals("followUps")) { fieldValue = new Gson().toJson(fieldValue); @@ -89,6 +97,42 @@ public static Map transformObjectToMap(Class aClass, Object obj) return map; } + private static Map dynamicFieldsSpecialCase(Object fieldValue) { + Map dynamicMap = Map.of(); + if (isJsonInString(fieldValue)) { + String strValue = (String) fieldValue; + ObjectMapper objectMapper = new ObjectMapper(); + try { + dynamicMap = objectMapper.readValue(strValue, Map.class); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return dynamicMap; + } + + private static boolean isJsonInString(Object fieldValue) { + return fieldValue instanceof String && StringUtils.isNotBlank((String) fieldValue) && isJson((String) fieldValue); + } + + private static boolean isJson(String str) { + return getFirstChar(str) == '{' && getLastChar(str) == '}'; + } + + private static char getLastChar(String strValue) { + if (Objects.isNull(strValue) || strValue.length() == 0) { + throw new IllegalArgumentException(); + } + return strValue.charAt(strValue.length() - 1); + } + + private static char getFirstChar(String strValue) { + if (Objects.isNull(strValue) || strValue.length() == 0) { + throw new IllegalArgumentException(); + } + return strValue.charAt(0); + } + public static class Constants { public static final String PROFILE = "profile"; public static final String PROFILE_HRUID = PROFILE + ".hruid"; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java index 3e9bd0191..d2001a86f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java @@ -1,17 +1,20 @@ package org.broadinstitute.dsm.model.elastic.migrationscript; -import org.broadinstitute.dsm.db.OncHistoryDetail; -import org.broadinstitute.dsm.db.ParticipantData; -import org.broadinstitute.dsm.statics.ESObjectConstants; - import java.util.List; import java.util.Map; +import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; +import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; +import org.broadinstitute.dsm.statics.ESObjectConstants; + public class ParticipantDataMigrate extends BaseMigrator { + private ParticipantDataDao participantDataDao; + public ParticipantDataMigrate(String index, String realm) { - super(index, realm, ESObjectConstants.PARTICIPANT_DATA, "participantDataId", ParticipantData.class); + super(index, realm, ESObjectConstants.PARTICIPANT_DATA, "participantDataId", ParticipantDataDto.class); + participantDataDao = new ParticipantDataDao(); } @Override From 8997e9ca84ff73026f179dc6115f06aaf07fec77 Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 9 Nov 2021 18:02:40 +0400 Subject: [PATCH 103/405] DDP-7117 | create method based on Special Case Pattern in Util.java, add test after refactoring to make sure that behaviour is kept --- .../ddp/participant/ParticipantDataDto.java | 1 + .../dsm/model/elastic/Util.java | 39 ++++++++++++++----- .../elastic/migrationscript/BaseMigrator.java | 2 +- .../migrationscript/ParticipantMigrate.java | 3 +- .../dsm/model/elastic/UtilTest.java | 19 ++++++++- .../MedicalRecordMigrateTest.java | 4 +- 6 files changed, 53 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java b/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java index 2fc4fae5d..16a6b285e 100644 --- a/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java +++ b/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java @@ -29,6 +29,7 @@ public class ParticipantDataDto { @ColumnName(ParticipantDataDao.FIELD_TYPE_ID) private String fieldTypeId; + @ColumnName(ParticipantDataDao.DATA) private String data; private long lastChanged; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index d8bbfc711..69081165c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -62,17 +62,17 @@ public static String underscoresToCamelCase(String fieldName) { return String.join("", words); } - public static List> transformObjectCollectionToCollectionMap(List values, Class aClass) { + public static List> transformObjectCollectionToCollectionMap(List values) { List> result = new ArrayList<>(); for (Object obj : values) { - result.add(transformObjectToMap(aClass, obj)); + result.add(transformObjectToMap(obj)); } return result; } - public static Map transformObjectToMap(Class aClass, Object obj) { + public static Map transformObjectToMap(Object obj) { Map map = new HashMap<>(); - Field[] declaredFields = aClass.getDeclaredFields(); + Field[] declaredFields = obj.getClass().getDeclaredFields(); for (Field declaredField : declaredFields) { ColumnName annotation = declaredField.getAnnotation(ColumnName.class); if (annotation == null) { @@ -84,12 +84,7 @@ public static Map transformObjectToMap(Class aClass, Object obj) if (Objects.isNull(fieldValue)) { continue; } - dynamicFieldsSpecialCase(fieldValue); - String key = underscoresToCamelCase(annotation.value()); - if (key.equals("followUps")) { - fieldValue = new Gson().toJson(fieldValue); - } - map.put(key, fieldValue); + map.putAll(convertToMap(annotation.value(), fieldValue)); } catch (IllegalAccessException e) { throw new RuntimeException(e); } @@ -97,6 +92,30 @@ public static Map transformObjectToMap(Class aClass, Object obj) return map; } + private static Map convertToMap(String fieldName, Object fieldValue) { + Map finalResult; + switch (fieldName) { + case "followUps": + finalResult = Map.of(underscoresToCamelCase(fieldName), new Gson().toJson(fieldValue)); + break; + case "data": + Map objectMap = dynamicFieldsSpecialCase(fieldValue); + Map transformedMap = new HashMap<>(); + for (Map.Entry object: objectMap.entrySet()) { + String field = (String) object.getKey(); + Object value = object.getValue(); + String camelCaseField = underscoresToCamelCase(field); + transformedMap.put(camelCaseField, value); + } + finalResult = transformedMap; + break; + default: + finalResult = Map.of(underscoresToCamelCase(fieldName), fieldValue); + break; + } + return finalResult; + } + private static Map dynamicFieldsSpecialCase(Object fieldValue) { Map dynamicMap = Map.of(); if (isJsonInString(fieldValue)) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java index 11d399c6e..7755f4da1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java @@ -49,7 +49,7 @@ protected void fillBulkRequestWithTransformedMap(Map> parti List recordList = entry.getValue(); participantId = getParticipantGuid(participantId, index); if (StringUtils.isBlank(participantId)) continue; - transformedList = Util.transformObjectCollectionToCollectionMap(recordList, this.aClass); + transformedList = Util.transformObjectCollectionToCollectionMap(recordList); setPrimaryId(); bulkExportFacade.addDataToRequest(generate(), participantId); System.err.println(participantId); //FOR TESTING diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrate.java index 6623ed8f8..b6196fbea 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrate.java @@ -29,9 +29,10 @@ public void export() { for (Map.Entry entry: participants.entrySet()) { Participant participant = entry.getValue(); String participantId = entry.getKey(); - transformedObject = Util.transformObjectToMap(Participant.class, participant); + transformedObject = Util.transformObjectToMap(participant); bulkExportFacade.addDataToRequest(generate(), getParticipantGuid(participantId, index)); } + bulkExportFacade.executeBulkUpsert(); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java index 432412d9b..ee68bc0aa 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java @@ -5,6 +5,7 @@ import java.util.Map; import org.broadinstitute.dsm.db.Participant; +import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; import org.junit.Test; public class UtilTest { @@ -33,7 +34,7 @@ public void transformObjectToMap() { "2020-10-28", "2020-10-28", "2020-10-28", "ptNotes", true, true, "additionalValuesJson", 1934283746283L); - Map transformedObject = Util.transformObjectToMap(Participant.class, participant); + Map transformedObject = Util.transformObjectToMap(participant); assertEquals("1", transformedObject.get("participantId")); assertEquals("QWERTY", transformedObject.get("ddpParticipantId")); assertEquals("2020-10-28", transformedObject.get("created")); @@ -41,4 +42,20 @@ public void transformObjectToMap() { assertEquals(1934283746283L, transformedObject.get("exitDate")); } + @Test + public void transformJsonToMap() { + String json = "{\"DDP_INSTANCE\": \"TEST\", \"DDP_VALUE\": \"VALUE\"}"; + + ParticipantDataDto participantDataDto = new ParticipantDataDto.Builder() + .withParticipantDataId(10) + .withDdpParticipantId("123") + .withDdpInstanceId(55) + .withFieldTypeId("f") + .withData(json) + .build(); + + Map result = Util.transformObjectToMap(participantDataDto); + assertEquals("TEST", result.get("ddpInstance")); + assertEquals("VALUE", result.get("ddpValue")); + } } \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java index 3034384bd..d22a05a00 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java @@ -23,7 +23,7 @@ public static void setUp() { @Test public void transformMedicalRecordToMap() { List medicalRecords = Arrays.asList(new MedicalRecord("1", "2", "3", "TYPE")); - List> listOfMaps = Util.transformObjectCollectionToCollectionMap(medicalRecords, MedicalRecord.class); + List> listOfMaps = Util.transformObjectCollectionToCollectionMap(medicalRecords); Map stringObjectMap = listOfMaps.get(0); Assert.assertEquals("1", stringObjectMap.get("medicalRecordId")); Assert.assertEquals("2", stringObjectMap.get("institutionId")); @@ -34,7 +34,7 @@ public void transformMedicalRecordToMap() { @Test public void generateSource() { List medicalRecords = Arrays.asList(new MedicalRecord("1", "2", "3", "TYPE")); - List> listOfMaps = Util.transformObjectCollectionToCollectionMap(medicalRecords, MedicalRecord.class); + List> listOfMaps = Util.transformObjectCollectionToCollectionMap(medicalRecords); MedicalRecordMigrate migrator = new MedicalRecordMigrate("", ""); Map resultMap = migrator.generate(); Map dsm = (Map)resultMap.get("dsm"); From d467e4b1057cb8a4467e20cca92837325a328237 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 9 Nov 2021 18:54:52 +0400 Subject: [PATCH 104/405] DDP-7117 | start creating of mapper for dynamic fields --- .../ParticipantDataMigrate.java | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java index d2001a86f..2c9de757b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java @@ -1,9 +1,12 @@ package org.broadinstitute.dsm.model.elastic.migrationscript; +import java.io.IOException; +import java.lang.reflect.Field; import java.util.List; import java.util.Map; +import com.fasterxml.jackson.databind.ObjectMapper; import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; import org.broadinstitute.dsm.statics.ESObjectConstants; @@ -19,6 +22,35 @@ public ParticipantDataMigrate(String index, String realm) { @Override protected Map> getDataByRealm() { - return null; + + Map> participantDataByRealm = participantDataDao.getParticipantDataByRealm(realm); + return (Map) participantDataByRealm; + } + + @Override + public void export() { + //decorate here + Map> dataByRealm = getDataByRealm(); + try { + Field dataField = ParticipantDataDto.class.getField("data"); + dataField.setAccessible(true); + for (Map.Entry> entry: dataByRealm.entrySet()) { + String participantId = entry.getKey(); + List datas = entry.getValue(); + for (Object data: datas) { + ParticipantDataDto pDataDto = (ParticipantDataDto) data; + String json = (String) dataField.get(pDataDto); + ObjectMapper objectMapper = new ObjectMapper(); + try { + objectMapper.readValue(json, Map.class); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new RuntimeException(e); + } + super.export(); } } From 3460a9366f29547fa547ed7393d2f6314c9e4706 Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 10 Nov 2021 11:19:41 +0400 Subject: [PATCH 105/405] DDP-7117 | continue ParticipantMigrate refactoring --- .../ddp/participant/ParticipantDataDao.java | 2 +- .../ParticipantDataMigrate.java | 61 ++++++++++++++++--- .../ParticipantDataMigrateTest.java | 19 ++++++ 3 files changed, 72 insertions(+), 10 deletions(-) create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrateTest.java diff --git a/src/main/java/org/broadinstitute/dsm/db/dao/ddp/participant/ParticipantDataDao.java b/src/main/java/org/broadinstitute/dsm/db/dao/ddp/participant/ParticipantDataDao.java index 21363100a..f268d4052 100644 --- a/src/main/java/org/broadinstitute/dsm/db/dao/ddp/participant/ParticipantDataDao.java +++ b/src/main/java/org/broadinstitute/dsm/db/dao/ddp/participant/ParticipantDataDao.java @@ -61,7 +61,7 @@ public class ParticipantDataDao implements Dao { "last_changed = ?," + "changed_by = ?"; - public static final String SQL_SELECT_PARTICIPANTS_BY_REALM = "SELECT d.participant_data_id, d.ddp_participant_id, d.field_type_id, d.data " + + public static final String SQL_SELECT_PARTICIPANTS_BY_REALM = "SELECT d.participant_data_id, d.ddp_instance_id, d.ddp_participant_id, d.field_type_id, d.data " + "FROM ddp_participant_data d " + "LEFT JOIN ddp_instance realm on (d.ddp_instance_id = realm.ddp_instance_id) " + "WHERE realm.instance_name = ? "; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java index 2c9de757b..c3297b5a1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java @@ -3,14 +3,24 @@ import java.io.IOException; import java.lang.reflect.Field; -import java.util.List; -import java.util.Map; +import java.util.*; import com.fasterxml.jackson.databind.ObjectMapper; import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; +import org.broadinstitute.dsm.model.NameValue; +import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.model.elastic.export.ElasticMappingExportAdapter; +import org.broadinstitute.dsm.model.elastic.export.UpsertMappingRequestPayload; +import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; +import org.broadinstitute.dsm.model.elastic.export.generate.MappingGenerator; +import org.broadinstitute.dsm.model.elastic.export.parse.BaseParser; +import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; import org.broadinstitute.dsm.statics.ESObjectConstants; +import static org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator.DSM_OBJECT; +import static org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator.PROPERTIES; + public class ParticipantDataMigrate extends BaseMigrator { private ParticipantDataDao participantDataDao; @@ -29,20 +39,44 @@ protected Map> getDataByRealm() { @Override public void export() { - //decorate here + exportMapping(); + super.export(); + } + + private void exportMapping() { + + Map mapping = new HashMap<>(); Map> dataByRealm = getDataByRealm(); + try { - Field dataField = ParticipantDataDto.class.getField("data"); + Field dataField = ParticipantDataDto.class.getDeclaredField("data"); + BaseParser typeParser = new TypeParser(); dataField.setAccessible(true); - for (Map.Entry> entry: dataByRealm.entrySet()) { + for (Map.Entry> entry : dataByRealm.entrySet()) { String participantId = entry.getKey(); + System.out.println(participantId); List datas = entry.getValue(); - for (Object data: datas) { - ParticipantDataDto pDataDto = (ParticipantDataDto) data; + for (Object data : datas) { + NameValue nameValue = new NameValue("d.data", dataField.get(data)); + GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, 0); + MappingGenerator mappingGenerator = new MappingGenerator(typeParser, generatorPayload); + + + mapping.merge("properties", mappingGenerator.generate(), (prev, curr) -> { + ((Map) prev).merge() + }); + + ParticipantDataDto pDataDto = (ParticipantDataDto) data; String json = (String) dataField.get(pDataDto); ObjectMapper objectMapper = new ObjectMapper(); try { - objectMapper.readValue(json, Map.class); + Map map = objectMapper.readValue(json, Map.class); + for (Map.Entry dynamicField : map.entrySet()) { + String camelCaseKey = Util.underscoresToCamelCase(dynamicField.getKey()); + + + mapping.put(camelCaseKey, typeParser.parse(String.valueOf(dynamicField.getValue()))); + } } catch (IOException e) { e.printStackTrace(); } @@ -51,6 +85,15 @@ public void export() { } catch (NoSuchFieldException | IllegalAccessException e) { throw new RuntimeException(e); } - super.export(); + + + Map innerMap = Map.of(PROPERTIES, mapping, "type", "nested"); + Map objectLevel = Map.of("participantData", innerMap); + Map dsmLevelProperties = Map.of(PROPERTIES, objectLevel); + Map> dsmLevel = Map.of(DSM_OBJECT, dsmLevelProperties); + Map finalMapping = Map.of(PROPERTIES, dsmLevel); + + ElasticMappingExportAdapter mappingExporter = new ElasticMappingExportAdapter(new UpsertMappingRequestPayload("participants_structured.atcp.atcp"), finalMapping); + mappingExporter.export(); } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrateTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrateTest.java new file mode 100644 index 000000000..068b51435 --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrateTest.java @@ -0,0 +1,19 @@ +package org.broadinstitute.dsm.model.elastic.migrationscript; + +import org.broadinstitute.dsm.TestHelper; +import org.junit.BeforeClass; +import org.junit.Test; + +public class ParticipantDataMigrateTest { + + @BeforeClass + public static void setUp() { + TestHelper.setupDB(); + } + + @Test + public void export() { + BaseMigrator participantMigrator = new ParticipantDataMigrate("participants_structured.atcp.atcp", "atcp"); + participantMigrator.export(); + } +} \ No newline at end of file From f39573b21d86f06b0be6297569f3829e4eb16f2e Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 10 Nov 2021 17:20:28 +0400 Subject: [PATCH 106/405] DDP-7117 | introduce merger abstraction, refactor existing code --- .../dsm/model/elastic/Util.java | 13 ++++- .../export/generate/GeneratorPayload.java | 4 ++ .../export/generate/MappingGenerator.java | 28 +++++++--- .../model/elastic/export/generate/Merger.java | 9 ++++ .../export/generate/SourceGenerator.java | 6 +-- .../ParticipantDataMigrate.java | 52 ++++--------------- .../dsm/model/elastic/UtilTest.java | 4 ++ .../export/generate/MappingGeneratorTest.java | 28 +++++++++- 8 files changed, 90 insertions(+), 54 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Merger.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 69081165c..a22120bf1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -30,6 +30,7 @@ public class Util { public static final String UNDERSCORE_SEPARATOR = "_"; public static final String DOC = "_doc"; public static final String ID = "id"; + public static final String CAMEL_CASE_REGEX = "(([a-z])+([A-z]))*"; public static String getQueryTypeFromId(String id) { String type; @@ -50,7 +51,9 @@ public static DBElement getDBElement(String fieldName) { } public static String underscoresToCamelCase(String fieldName) { - List words = Arrays.stream(fieldName.split(UNDERSCORE_SEPARATOR)) + String[] splittedWords = fieldName.split(UNDERSCORE_SEPARATOR); + if (hasNoUnderscores(splittedWords)) return handleAllUppercase(fieldName); + List words = Arrays.stream(splittedWords) .map(word -> new StringBuilder(word.toLowerCase())) .collect(Collectors.toList()); for (int i = FIRST_ELEMENT_INDEX; i < words.size(); i++) { @@ -62,6 +65,14 @@ public static String underscoresToCamelCase(String fieldName) { return String.join("", words); } + private static String handleAllUppercase(String word) { + return word.matches(CAMEL_CASE_REGEX) ? word : word.toLowerCase(); + } + + private static boolean hasNoUnderscores(String[] splittedWords) { + return splittedWords.length < 2; + } + public static List> transformObjectCollectionToCollectionMap(List values) { List> result = new ArrayList<>(); for (Object obj : values) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorPayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorPayload.java index 75ab7cdec..075389a3c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorPayload.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorPayload.java @@ -12,6 +12,10 @@ public GeneratorPayload(NameValue nameValue, int recordId) { this.recordId = recordId; } + public GeneratorPayload(NameValue nameValue) { + this.nameValue = nameValue; + } + public NameValue getNameValue() { return nameValue; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java index 3efaedf13..5af230e0e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java @@ -8,7 +8,7 @@ import java.util.HashMap; import java.util.Map; -public class MappingGenerator extends BaseGenerator { +public class MappingGenerator extends BaseGenerator implements Merger { private static final Logger logger = LoggerFactory.getLogger(MappingGenerator.class); @@ -42,7 +42,7 @@ protected Map parseJson() { Map fieldsByValues = parseJsonToMapFromValue(); for (Map.Entry entry: fieldsByValues.entrySet()) { Object eachType = parser.parse(String.valueOf(entry.getValue())); - resultMap.put(entry.getKey(), eachType); + resultMap.put(Util.underscoresToCamelCase(entry.getKey()), eachType); } return resultMap; } @@ -56,23 +56,39 @@ protected Object parseSingleElement() { protected Map getElementWithId(Object type) { return Map.of( Util.ID, Map.of(TYPE, TYPE_KEYWORD), - getDBElement().getColumnName(), type + Util.underscoresToCamelCase(getDBElement().getColumnName()), type ); } @Override protected Map getElement(Object type) { - return Map.of(getDBElement().getColumnName(), type); + return Map.of(Util.underscoresToCamelCase(getDBElement().getColumnName()), type); } @Override protected Object constructSingleElement() { - return Map.of(PROPERTIES, collect()); + return new HashMap<>(Map.of(PROPERTIES, collect())); } @Override protected Object constructCollection() { - return Map.of(TYPE, NESTED, PROPERTIES, collect()); + return new HashMap<>(Map.of(TYPE, NESTED, PROPERTIES, collect())); + } + + @Override + public Map merge(Map base, Map toMerge) { + base.putIfAbsent(PROPERTIES, toMerge.get(PROPERTIES)); + getFieldLevel(base).putAll(getFieldLevel(toMerge)); + return base; + } + + private Map getFieldLevel(Map topLevel) { + String propertyName = getOuterPropertyByAlias().getPropertyName(); + return ((Map)((Map)((Map)((Map)((Map)topLevel.get(PROPERTIES)) + .get(DSM_OBJECT)) + .get(PROPERTIES)) + .get(propertyName)) + .get(PROPERTIES)); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Merger.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Merger.java new file mode 100644 index 000000000..7f5c71454 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Merger.java @@ -0,0 +1,9 @@ +package org.broadinstitute.dsm.model.elastic.export.generate; + +import java.util.Map; + +public interface Merger { + + Map merge(Map base, Map toMerge); + +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java index 676769bc3..b1be50015 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java @@ -39,7 +39,7 @@ private Map parseJsonValuesToObject() { Map dynamicFieldValues = parseJsonToMapFromValue(); Map transformedMap = new HashMap<>(); for (Map.Entry entry : dynamicFieldValues.entrySet()) { - transformedMap.put(entry.getKey(), parser.parse(String.valueOf(entry.getValue()))); + transformedMap.put(Util.underscoresToCamelCase(entry.getKey()), parser.parse(String.valueOf(entry.getValue()))); } return transformedMap; } @@ -55,7 +55,7 @@ protected Object getElementWithId(Object element) { } private List> buildCollectionWithId(Object element) { - return List.of(Map.of(getDBElement().getColumnName(), element, + return List.of(Map.of(Util.underscoresToCamelCase(getDBElement().getColumnName()), element, Util.ID, generatorPayload.getRecordId())); } @@ -66,7 +66,7 @@ protected Map getElement(Object element) { private Map buildSingleFieldWithValue(Object element) { logger.info("Constructing single field with value"); - return Map.of(getDBElement().getColumnName(), element); + return Map.of(Util.underscoresToCamelCase(getDBElement().getColumnName()), element); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java index c3297b5a1..c174c3426 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java @@ -1,15 +1,12 @@ package org.broadinstitute.dsm.model.elastic.migrationscript; -import java.io.IOException; import java.lang.reflect.Field; import java.util.*; -import com.fasterxml.jackson.databind.ObjectMapper; import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; import org.broadinstitute.dsm.model.NameValue; -import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.ElasticMappingExportAdapter; import org.broadinstitute.dsm.model.elastic.export.UpsertMappingRequestPayload; import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; @@ -18,11 +15,9 @@ import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; import org.broadinstitute.dsm.statics.ESObjectConstants; -import static org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator.DSM_OBJECT; -import static org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator.PROPERTIES; - public class ParticipantDataMigrate extends BaseMigrator { + public static final String DATA_WITH_ALIAS = "d.data"; private ParticipantDataDao participantDataDao; public ParticipantDataMigrate(String index, String realm) { @@ -47,53 +42,26 @@ private void exportMapping() { Map mapping = new HashMap<>(); Map> dataByRealm = getDataByRealm(); + BaseParser typeParser = new TypeParser(); try { - Field dataField = ParticipantDataDto.class.getDeclaredField("data"); - BaseParser typeParser = new TypeParser(); + Field dataField = ParticipantDataDto.class.getDeclaredField(ParticipantDataDao.DATA); dataField.setAccessible(true); for (Map.Entry> entry : dataByRealm.entrySet()) { - String participantId = entry.getKey(); - System.out.println(participantId); - List datas = entry.getValue(); - for (Object data : datas) { - NameValue nameValue = new NameValue("d.data", dataField.get(data)); - GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, 0); + List participantDatas = entry.getValue(); + for (Object jsonData : participantDatas) { + NameValue nameValue = new NameValue(DATA_WITH_ALIAS, dataField.get(jsonData)); + GeneratorPayload generatorPayload = new GeneratorPayload(nameValue); MappingGenerator mappingGenerator = new MappingGenerator(typeParser, generatorPayload); - - - mapping.merge("properties", mappingGenerator.generate(), (prev, curr) -> { - ((Map) prev).merge() - }); - - ParticipantDataDto pDataDto = (ParticipantDataDto) data; - String json = (String) dataField.get(pDataDto); - ObjectMapper objectMapper = new ObjectMapper(); - try { - Map map = objectMapper.readValue(json, Map.class); - for (Map.Entry dynamicField : map.entrySet()) { - String camelCaseKey = Util.underscoresToCamelCase(dynamicField.getKey()); - - - mapping.put(camelCaseKey, typeParser.parse(String.valueOf(dynamicField.getValue()))); - } - } catch (IOException e) { - e.printStackTrace(); - } + mapping = mappingGenerator.merge(mapping, mappingGenerator.generate()); } } } catch (NoSuchFieldException | IllegalAccessException e) { throw new RuntimeException(e); } - - Map innerMap = Map.of(PROPERTIES, mapping, "type", "nested"); - Map objectLevel = Map.of("participantData", innerMap); - Map dsmLevelProperties = Map.of(PROPERTIES, objectLevel); - Map> dsmLevel = Map.of(DSM_OBJECT, dsmLevelProperties); - Map finalMapping = Map.of(PROPERTIES, dsmLevel); - - ElasticMappingExportAdapter mappingExporter = new ElasticMappingExportAdapter(new UpsertMappingRequestPayload("participants_structured.atcp.atcp"), finalMapping); + ElasticMappingExportAdapter mappingExporter = new ElasticMappingExportAdapter(new UpsertMappingRequestPayload( + index), mapping); mappingExporter.export(); } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java index ee68bc0aa..f379543ab 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java @@ -16,16 +16,20 @@ public void underscoresToCamelCase() { String fieldName2 = "COLUMN_NAME"; String fieldName3 = "column"; String fieldName4 = "COLUMN"; + String fieldName5 = "columnName"; String transformed = Util.underscoresToCamelCase(fieldName); String transformed2 = Util.underscoresToCamelCase(fieldName2); String transformed3 = Util.underscoresToCamelCase(fieldName3); String transformed4 = Util.underscoresToCamelCase(fieldName4); + String transformed5 = Util.underscoresToCamelCase(fieldName5); assertEquals("columnName", transformed); assertEquals("columnName", transformed2); assertEquals("column", transformed3); assertEquals("column", transformed4); + assertEquals("columnName", transformed5); } + @Test public void transformObjectToMap() { Participant participant = new Participant( diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java index 02a2e49b2..36b560f67 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java @@ -10,7 +10,9 @@ import org.junit.Assert; import org.junit.Test; +import java.util.HashMap; import java.util.Map; +import java.util.Objects; public class MappingGeneratorTest { @@ -104,13 +106,35 @@ public void generateMapping() { Assert.assertTrue(topLevel.equals(resultMap)); } + @Test + public void merge() { + NameValue nameValue = new NameValue("d.data", "{\"DDP_INSTANCE\": \"TEST\"}"); + GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, 0); + TypeParser parser = new TypeParser(); + MappingGenerator mappingGenerator = new MappingGenerator(parser, generatorPayload); + Map base = new HashMap<>(); + base = mappingGenerator.merge(base, mappingGenerator.generate()); + NameValue nameValue2 = new NameValue("d.data", "{\"DDP_INSTANCE1\": \"TEST1\"}"); + GeneratorPayload generatorPayload2 = new GeneratorPayload(nameValue2, 0); + mappingGenerator = new MappingGenerator(parser, generatorPayload2); + base = mappingGenerator.merge(base, mappingGenerator.generate()); + Object value = ((Map) ((Map) ((Map) ((Map) ((Map) base + .get(MappingGenerator.PROPERTIES)) + .get(MappingGenerator.DSM_OBJECT)) + .get(MappingGenerator.PROPERTIES)) + .get(mappingGenerator.getOuterPropertyByAlias().getPropertyName())) + .get(MappingGenerator.PROPERTIES)) + .get("DDP_INSTANCE1"); + Assert.assertFalse(Objects.isNull(value)); + } + private String extractDeepestLeveleValue(Map objectMap) { return (String) ((Map) ((Map) getMedicalRecordProperty(objectMap) .get(BaseGenerator.PROPERTIES)) - .get("medical_record_column")) + .get("medicalRecordColumn")) .get("type"); } @@ -122,7 +146,7 @@ private String extractKeywordType(Map objectMap) { ((Map) getMedicalRecordProperty(objectMap) .get(BaseGenerator.PROPERTIES)) - .get("medical_record_column")) + .get("medicalRecordColumn")) .get("fields")) .get("keyword")) .get("type"); From e53b937005ad154ddcf7b014627b380acc942049 Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 10 Nov 2021 18:54:57 +0400 Subject: [PATCH 107/405] DDP-7117 | finalize KitRequestShippingMigrate, add new constants to DBConstants, add appropriate annotations --- .../dsm/db/KitRequestShipping.java | 20 +++++ .../broadinstitute/dsm/model/KitRequest.java | 3 + .../dsm/model/elastic/Util.java | 4 +- .../KitRequestShippingMigrate.java | 81 ++++--------------- .../ParticipantDataMigrate.java | 4 - .../dsm/statics/DBConstants.java | 3 +- .../dsm/statics/ESObjectConstants.java | 3 + .../KitRequestShippingMigrateTest.java | 20 +++++ 8 files changed, 65 insertions(+), 73 deletions(-) create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrateTest.java diff --git a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java index a2000d714..73dc24549 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java @@ -119,6 +119,8 @@ public class KitRequestShipping extends KitRequest { public static final String DEACTIVATION_REASON = "Generated Express"; private static final String QUEUE = "queue"; + + @ColumnName(DBConstants.ERROR) private static final String ERROR = "error"; private static final String SENT = "sent"; private static final String RECEIVED = "received"; @@ -139,8 +141,13 @@ public class KitRequestShipping extends KitRequest { private static final String SEARCH_TRACKING_NUMBER = "TRACKING_NUMBER"; private static final String SEARCH_MF_BAR = "MF_BAR"; + @ColumnName(DBConstants.DSM_KIT_ID) private final String dsmKitId; + + @ColumnName(DBConstants.LABEL_URL_TO) private final String labelUrlTo; + + @ColumnName(DBConstants.LABEL_URL_RETURN) private final String labelUrlReturn; @ColumnName (DBConstants.DSM_TRACKING_TO) @@ -157,11 +164,14 @@ public class KitRequestShipping extends KitRequest { @ColumnName (DBConstants.BSP_COLLABORATOR_PARTICIPANT_ID) private final String bspCollaboratorSampleId; + private final String easypostAddressId; private final String realm; @ColumnName (DBConstants.KIT_TYPE_NAME) private final String kitType; + + @ColumnName (DBConstants.DEACTIVATION_REASON) private final String deactivationReason; @ColumnName (DBConstants.KIT_LABEL) @@ -181,19 +191,29 @@ public class KitRequestShipping extends KitRequest { @ColumnName (DBConstants.DSM_DEACTIVATED_DATE) @DbDateConversion(SqlDateConverter.EPOCH) private final long deactivatedDate; + + @ColumnName (DBConstants.EXPRESS) private final boolean express; + + @ColumnName (DBConstants.EASYPOST_TO_ID) private final String easypostToId; private final long labelTriggeredDate; + + @ColumnName (DBConstants.NO_RETURN) private final boolean noReturn; @ColumnName (DBConstants.ERROR) private boolean error; + + @ColumnName(DBConstants.MESSAGE) private String message; + @ColumnName (DBConstants.EASYPOST_SHIPMENT_STATUS) private String easypostShipmentStatus; private String nameLabel; + @ColumnName (DBConstants.CREATED_BY) private String createdBy; private String preferredLanguage; private String hruid; diff --git a/src/main/java/org/broadinstitute/dsm/model/KitRequest.java b/src/main/java/org/broadinstitute/dsm/model/KitRequest.java index 222bfeef4..59600aa12 100644 --- a/src/main/java/org/broadinstitute/dsm/model/KitRequest.java +++ b/src/main/java/org/broadinstitute/dsm/model/KitRequest.java @@ -16,11 +16,14 @@ columnPrefix = "") public class KitRequest { + @ColumnName(DBConstants.DSM_KIT_REQUEST_ID) private String dsmKitRequestId; private String participantId; private String shortId; private String shippingId; private DDPParticipant participant; + + @ColumnName(DBConstants.EXTERNAL_ORDER_STATUS) private String externalOrderStatus; private String externalKitName; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index a22120bf1..649189192 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -83,7 +83,9 @@ public static List> transformObjectCollectionToCollectionMap public static Map transformObjectToMap(Object obj) { Map map = new HashMap<>(); - Field[] declaredFields = obj.getClass().getDeclaredFields(); + List declaredFields = new ArrayList(List.of(obj.getClass().getDeclaredFields())); + List declaredFieldsSuper = new ArrayList(List.of(obj.getClass().getSuperclass().getDeclaredFields())); + declaredFields.addAll(declaredFieldsSuper); for (Field declaredField : declaredFields) { ColumnName annotation = declaredField.getAnnotation(ColumnName.class); if (annotation == null) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrate.java index 880a8655b..ac581658a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrate.java @@ -1,79 +1,26 @@ package org.broadinstitute.dsm.model.elastic.migrationscript; +import org.broadinstitute.dsm.db.DDPInstance; +import org.broadinstitute.dsm.db.KitRequestShipping; +import org.broadinstitute.dsm.statics.ESObjectConstants; + import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.BOOLEAN_MAPPING; import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.DATE_MAPPING; import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.TEXT_KEYWORD_MAPPING; +import java.util.List; import java.util.Map; -public class KitRequestShippingMigrate { - - private static final Map kitRequestShippingMigrate1 = Map.of ( - "kitTypeName", TEXT_KEYWORD_MAPPING, - "instanceName", TEXT_KEYWORD_MAPPING, - "bspCollaboratorParticipantId", TEXT_KEYWORD_MAPPING, - "bspCollaboratorSampleId", TEXT_KEYWORD_MAPPING, - "ddpParticipantId", TEXT_KEYWORD_MAPPING, - "ddpLabel", TEXT_KEYWORD_MAPPING, - "dsmKitRequest", TEXT_KEYWORD_MAPPING, - "kitTypeId", TEXT_KEYWORD_MAPPING, - "externalOrderStatus", TEXT_KEYWORD_MAPPING, - "externalOrderNumber", TEXT_KEYWORD_MAPPING - ); - - private static final Map kitRequestShippingMigrate2 = Map.of ( - "externalOrderDate", TEXT_KEYWORD_MAPPING, - "externalResponse", TEXT_KEYWORD_MAPPING, - "uploadReason", TEXT_KEYWORD_MAPPING, - "noReturn", BOOLEAN_MAPPING, - "createdBy", TEXT_KEYWORD_MAPPING, - "dsmKitRequestId", TEXT_KEYWORD_MAPPING, - "dsmKitId", TEXT_KEYWORD_MAPPING, - "kitComplete", BOOLEAN_MAPPING, - "labelUrlTo", TEXT_KEYWORD_MAPPING, - "labelUrlReturn", TEXT_KEYWORD_MAPPING - ); - - private static final Map kitRequestShippingMigrate3 = Map.of ( - "trackingToId", TEXT_KEYWORD_MAPPING, - "trackingReturnId", TEXT_KEYWORD_MAPPING, - "easypostTrackingToUrl", TEXT_KEYWORD_MAPPING, - "easypostTrackingReturnUrl", TEXT_KEYWORD_MAPPING, - "easypostToId", TEXT_KEYWORD_MAPPING, - "easypostShipmentStatus", BOOLEAN_MAPPING, - "scanDate", TEXT_KEYWORD_MAPPING, - "labelDate", TEXT_KEYWORD_MAPPING, - "error", BOOLEAN_MAPPING, - "message", TEXT_KEYWORD_MAPPING); - - private static final Map kitRequestShippingMigrate4 = Map.of ( - "receiveDate", TEXT_KEYWORD_MAPPING, - "deactivatedDate", TEXT_KEYWORD_MAPPING, - "easypostAddressIdTo", TEXT_KEYWORD_MAPPING, - "deactivationReason", TEXT_KEYWORD_MAPPING, - "trackingId", TEXT_KEYWORD_MAPPING, - "kitLabel", TEXT_KEYWORD_MAPPING, - "express", BOOLEAN_MAPPING, - "testResult", TEXT_KEYWORD_MAPPING, - "needsApproval", BOOLEAN_MAPPING, - "authorization", TEXT_KEYWORD_MAPPING - ); - - private static final Map kitRequestShippingMigrate5 = Map.of ( - "denialReason", TEXT_KEYWORD_MAPPING, - "authorizedBy", TEXT_KEYWORD_MAPPING, - "upsTrackingStatus", TEXT_KEYWORD_MAPPING, - "upsReturnStatus", TEXT_KEYWORD_MAPPING, - "CEOrder", BOOLEAN_MAPPING // ? what is this ? - ); +public class KitRequestShippingMigrate extends BaseMigrator { - private static final Map KitRequestShippingMerged = MapAdapter.of( - kitRequestShippingMigrate1, - kitRequestShippingMigrate2, - kitRequestShippingMigrate3, - kitRequestShippingMigrate4, - kitRequestShippingMigrate5 - ); + public KitRequestShippingMigrate(String index, String realm) { + super(index, realm, ESObjectConstants.KIT_REQUEST_SHIPPING, "dsmKitRequestId", KitRequestShipping.class); + } + @Override + protected Map> getDataByRealm() { + Map> kitRequests = KitRequestShipping.getAllKitRequestsByRealm(realm, null, null, true); + return (Map) kitRequests; + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java index c174c3426..87fe75d31 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java @@ -27,7 +27,6 @@ public ParticipantDataMigrate(String index, String realm) { @Override protected Map> getDataByRealm() { - Map> participantDataByRealm = participantDataDao.getParticipantDataByRealm(realm); return (Map) participantDataByRealm; } @@ -39,11 +38,9 @@ public void export() { } private void exportMapping() { - Map mapping = new HashMap<>(); Map> dataByRealm = getDataByRealm(); BaseParser typeParser = new TypeParser(); - try { Field dataField = ParticipantDataDto.class.getDeclaredField(ParticipantDataDao.DATA); dataField.setAccessible(true); @@ -59,7 +56,6 @@ private void exportMapping() { } catch (NoSuchFieldException | IllegalAccessException e) { throw new RuntimeException(e); } - ElasticMappingExportAdapter mappingExporter = new ElasticMappingExportAdapter(new UpsertMappingRequestPayload( index), mapping); mappingExporter.export(); diff --git a/src/main/java/org/broadinstitute/dsm/statics/DBConstants.java b/src/main/java/org/broadinstitute/dsm/statics/DBConstants.java index 37a16c5cf..4b81f9818 100644 --- a/src/main/java/org/broadinstitute/dsm/statics/DBConstants.java +++ b/src/main/java/org/broadinstitute/dsm/statics/DBConstants.java @@ -51,6 +51,7 @@ public class DBConstants { public static final String DSM_KIT_ID = "dsm_kit_id"; public static final String KIT_TYPE_NAME = "kit_type_name"; public static final String DSM_LABEL = "ddp_label"; + public static final String LABEL_URL_TO = "label_url_to"; public static final String KIT_TYPE_ID = "kit_type_id"; public static final String LAST_KIT = "last_kit"; public static final String DSM_KIT_REQUEST_ID = "dsm_kit_request_id"; @@ -482,5 +483,5 @@ public class DBConstants { public static final String PHONE_NUMBER = "phone_number"; - + public static final String LABEL_URL_RETURN = "label_url_return"; } diff --git a/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java b/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java index 3a37eca76..6487a2b9c 100644 --- a/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java +++ b/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java @@ -32,6 +32,9 @@ public class ESObjectConstants { public static final String RECEIVED = "received"; public static final String KIT_REQUEST_ID = "kitRequestId"; + // kit + public static final String KIT_REQUEST_SHIPPING = "kitRequestShipping"; + //common public static final String DDP_PARTICIPANT_ID = "ddpParticipantId"; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrateTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrateTest.java new file mode 100644 index 000000000..ff11cd7ad --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrateTest.java @@ -0,0 +1,20 @@ +package org.broadinstitute.dsm.model.elastic.migrationscript; + +import org.broadinstitute.dsm.TestHelper; +import org.junit.Before; +import org.junit.Test; + +public class KitRequestShippingMigrateTest { + + @Before + public void setUp() { + TestHelper.setupDB(); + } + + @Test + public void export() { + KitRequestShippingMigrate kitRequestShippingMigrate = new KitRequestShippingMigrate("participants_structured.cmi.cmi-brain", "brain"); + kitRequestShippingMigrate.export(); + } + +} \ No newline at end of file From 060acaefafdaddc439fb635e017e43d4a3ab74c1 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 11 Nov 2021 10:48:56 +0400 Subject: [PATCH 108/405] DDP-7117 | add pubsub triggering for ES exporting --- .../broadinstitute/dsm/export/ExportToES.java | 4 +++ .../elastic/migrationscript/MapAdapter.java | 16 --------- .../dsm/pubsub/DSMtasksSubscription.java | 33 +++++++++++++++++-- 3 files changed, 34 insertions(+), 19 deletions(-) delete mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MapAdapter.java diff --git a/src/main/java/org/broadinstitute/dsm/export/ExportToES.java b/src/main/java/org/broadinstitute/dsm/export/ExportToES.java index 6cbde4819..ba541089a 100644 --- a/src/main/java/org/broadinstitute/dsm/export/ExportToES.java +++ b/src/main/java/org/broadinstitute/dsm/export/ExportToES.java @@ -44,9 +44,13 @@ public void exportObjectsToES(String data, boolean clearBeforeUpdate) { public static class ExportPayload { private String study; + private boolean isNewExport; public String getStudy() { return study; } + public boolean isNewExport() { + return isNewExport; + } } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MapAdapter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MapAdapter.java deleted file mode 100644 index ecd902947..000000000 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MapAdapter.java +++ /dev/null @@ -1,16 +0,0 @@ -package org.broadinstitute.dsm.model.elastic.migrationscript; - -import java.util.HashMap; -import java.util.Map; - -public class MapAdapter { - - public static Map of(Map... maps) { - Map result = new HashMap<>(); - for (Map map: maps) { - result.putAll(map); - } - return result; - } - -} diff --git a/src/main/java/org/broadinstitute/dsm/pubsub/DSMtasksSubscription.java b/src/main/java/org/broadinstitute/dsm/pubsub/DSMtasksSubscription.java index 6b168ce3e..535adbae2 100644 --- a/src/main/java/org/broadinstitute/dsm/pubsub/DSMtasksSubscription.java +++ b/src/main/java/org/broadinstitute/dsm/pubsub/DSMtasksSubscription.java @@ -5,19 +5,30 @@ import com.google.cloud.pubsub.v1.AckReplyConsumer; import com.google.cloud.pubsub.v1.MessageReceiver; import com.google.cloud.pubsub.v1.Subscriber; +import com.google.gson.Gson; import com.google.pubsub.v1.ProjectSubscriptionName; import com.google.pubsub.v1.PubsubMessage; import org.apache.commons.lang3.StringUtils; +import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; +import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; import org.broadinstitute.dsm.export.ExportToES; import org.broadinstitute.dsm.model.Study; import org.broadinstitute.dsm.model.defaultvalues.Defaultable; import org.broadinstitute.dsm.model.defaultvalues.DefaultableMaker; +import org.broadinstitute.dsm.model.elastic.export.Exportable; +import org.broadinstitute.dsm.model.elastic.migrationscript.KitRequestShippingMigrate; +import org.broadinstitute.dsm.model.elastic.migrationscript.MedicalRecordMigrate; +import org.broadinstitute.dsm.model.elastic.migrationscript.OncHistoryDetailsMigrate; +import org.broadinstitute.dsm.model.elastic.migrationscript.ParticipantDataMigrate; +import org.broadinstitute.dsm.model.elastic.migrationscript.ParticipantMigrate; import org.broadinstitute.dsm.util.ParticipantUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Arrays; +import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -54,9 +65,25 @@ public static void subscribeDSMtasks(String projectId, String subscriptionId) { break; case ELASTIC_EXPORT: consumer.ack(); - boolean clearBeforeUpdate = attributesMap.containsKey(CLEAR_BEFORE_UPDATE) - && Boolean.parseBoolean(attributesMap.get(CLEAR_BEFORE_UPDATE)); - new ExportToES().exportObjectsToES(data, clearBeforeUpdate); + ExportToES.ExportPayload exportPayload = new Gson().fromJson(data, ExportToES.ExportPayload.class); + if (exportPayload.isNewExport()) { + String study = exportPayload.getStudy(); + Optional maybeDdpInstanceByInstanceName = + new DDPInstanceDao().getDDPInstanceByInstanceName(study); + maybeDdpInstanceByInstanceName.ifPresent(ddpInstanceDto -> { + String index = ddpInstanceDto.getEsParticipantIndex(); + List exportables = Arrays.asList(new MedicalRecordMigrate(index, study), + new OncHistoryDetailsMigrate(index, study), + new ParticipantDataMigrate(index, study), + new ParticipantMigrate(index, study), + new KitRequestShippingMigrate(index, study)); + exportables.forEach(Exportable::export); + }); + } else { + boolean clearBeforeUpdate = attributesMap.containsKey(CLEAR_BEFORE_UPDATE) + && Boolean.parseBoolean(attributesMap.get(CLEAR_BEFORE_UPDATE)); + new ExportToES().exportObjectsToES(data, clearBeforeUpdate); + } break; case PARTICIPANT_REGISTERED: generateStudyDefaultValues(consumer, attributesMap); From 099f3bcce823eb3d532cb3d41ee07d634fe68d57 Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 11 Nov 2021 16:45:59 +0400 Subject: [PATCH 109/405] DDP-7117 | initialize Pub/Sub functionality in order to trigger ES export --- .../broadinstitute/dsm/db/MedicalRecord.java | 1 + .../broadinstitute/dsm/export/ExportToES.java | 6 +- .../elastic/migrationscript/BaseMigrator.java | 90 ++++++++++++++++--- .../migrationscript/BulkExportFacade.java | 2 - .../dsm/pubsub/DSMtasksSubscription.java | 38 ++++---- 5 files changed, 102 insertions(+), 35 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java index b418f3339..18765f86a 100644 --- a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java +++ b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java @@ -441,6 +441,7 @@ public static Map> getMedicalRecords(@NonNull String while (rs.next()) { String ddpParticipantId = rs.getString(DBConstants.DDP_PARTICIPANT_ID); List medicalRecordList = new ArrayList<>(); + if (medicalRecords.keySet().size() > 15) break; if (medicalRecords.containsKey(ddpParticipantId)) { medicalRecordList = medicalRecords.get(ddpParticipantId); } diff --git a/src/main/java/org/broadinstitute/dsm/export/ExportToES.java b/src/main/java/org/broadinstitute/dsm/export/ExportToES.java index ba541089a..a1f2d53b3 100644 --- a/src/main/java/org/broadinstitute/dsm/export/ExportToES.java +++ b/src/main/java/org/broadinstitute/dsm/export/ExportToES.java @@ -44,13 +44,13 @@ public void exportObjectsToES(String data, boolean clearBeforeUpdate) { public static class ExportPayload { private String study; - private boolean isNewExport; + private boolean isMigration; public String getStudy() { return study; } - public boolean isNewExport() { - return isNewExport; + public boolean isMigration() { + return isMigration; } } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java index 7755f4da1..51c2b4e81 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java @@ -1,19 +1,31 @@ package org.broadinstitute.dsm.model.elastic.migrationscript; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ForkJoinTask; +import java.util.concurrent.RecursiveAction; +import org.broadinstitute.dsm.db.MedicalRecord; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.Exportable; import org.broadinstitute.dsm.model.elastic.export.generate.Generator; import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchable; +import org.broadinstitute.dsm.pubsub.DSMtasksSubscription; import org.broadinstitute.dsm.statics.ESObjectConstants; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import spark.utils.StringUtils; -public abstract class BaseMigrator implements Exportable, Generator { +public abstract class BaseMigrator extends RecursiveAction implements Exportable, Generator { + + private static final Logger logger = LoggerFactory.getLogger(BaseMigrator.class); private final ElasticSearchable elasticSearch; + public Map> dataByRealm; protected String object; protected final BulkExportFacade bulkExportFacade; private String primaryId; @@ -32,6 +44,35 @@ public BaseMigrator(String index, String realm, String object, String primaryId, this.aClass = aClass; } + @Override + protected void compute() { + if (dataByRealm.size() > 15) { + ForkJoinTask.invokeAll(createSubtasks()); + } else { + processing(dataByRealm); + } + } + + private void processing(Map> data) { + + + } + + private List createSubtasks() { + List subtasks = new ArrayList<>(); + + int low = 0; + + for (int i = 0; i < ; i++) { + + } + subtasks.add(new CustomRecursiveAction(partOne)); + subtasks.add(new CustomRecursiveAction(partTwo)); + + + return subtasks; + } + private void setPrimaryId() { for(Map map: transformedList) { map.put(Util.ID, map.get(primaryId)); @@ -43,24 +84,49 @@ public Map generate() { return Map.of(ESObjectConstants.DSM, Map.of(object, transformedList)); } - protected void fillBulkRequestWithTransformedMap(Map> participantRecords) { - for (Map.Entry> entry: participantRecords.entrySet()) { - String participantId = entry.getKey(); - List recordList = entry.getValue(); - participantId = getParticipantGuid(participantId, index); - if (StringUtils.isBlank(participantId)) continue; - transformedList = Util.transformObjectCollectionToCollectionMap(recordList); - setPrimaryId(); - bulkExportFacade.addDataToRequest(generate(), participantId); - System.err.println(participantId); //FOR TESTING + protected void fillBulkRequestWithTransformedMap() { + + if (participantRecords.size() < 15) { + for (Map.Entry> entry: participantRecords.entrySet()) { + String participantId = entry.getKey(); + List recordList = entry.getValue(); + participantId = getParticipantGuid(participantId, index); + if (StringUtils.isBlank(participantId)) continue; + transformedList = Util.transformObjectCollectionToCollectionMap(recordList); + setPrimaryId(); + bulkExportFacade.addDataToRequest(generate(), participantId); + System.err.println(participantId); //FOR TESTING + } + } else { + ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() - 1); + + + executorService.execute(() -> { + for (Map.Entry> entry: participantRecords.entrySet()) { + String participantId = entry.getKey(); + List recordList = entry.getValue(); + participantId = getParticipantGuid(participantId, index); + if (StringUtils.isBlank(participantId)) continue; + transformedList = Util.transformObjectCollectionToCollectionMap(recordList); + setPrimaryId(); + bulkExportFacade.addDataToRequest(generate(), participantId); + System.err.println(participantId); //FOR TESTING + } + }); } + } protected abstract Map> getDataByRealm(); @Override public void export() { - fillBulkRequestWithTransformedMap(getDataByRealm()); + dataByRealm = getDataByRealm(); + + fillBulkRequestWithTransformedMap(); + + bulkExportFacade.executeBulkUpsert(); + logger.info("finished migrating data to ES."); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BulkExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BulkExportFacade.java index 1a31546c2..35d1c5692 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BulkExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BulkExportFacade.java @@ -23,8 +23,6 @@ public BulkExportFacade(String index) { public void addDataToRequest(Map mapToUpsert, String docId) { bulkRequest.add(createRequest(mapToUpsert, docId)); - - } private UpdateRequest createRequest(Map mapToUpsert, String docId) { diff --git a/src/main/java/org/broadinstitute/dsm/pubsub/DSMtasksSubscription.java b/src/main/java/org/broadinstitute/dsm/pubsub/DSMtasksSubscription.java index 535adbae2..5ca34d12e 100644 --- a/src/main/java/org/broadinstitute/dsm/pubsub/DSMtasksSubscription.java +++ b/src/main/java/org/broadinstitute/dsm/pubsub/DSMtasksSubscription.java @@ -16,11 +16,7 @@ import org.broadinstitute.dsm.model.defaultvalues.Defaultable; import org.broadinstitute.dsm.model.defaultvalues.DefaultableMaker; import org.broadinstitute.dsm.model.elastic.export.Exportable; -import org.broadinstitute.dsm.model.elastic.migrationscript.KitRequestShippingMigrate; -import org.broadinstitute.dsm.model.elastic.migrationscript.MedicalRecordMigrate; -import org.broadinstitute.dsm.model.elastic.migrationscript.OncHistoryDetailsMigrate; -import org.broadinstitute.dsm.model.elastic.migrationscript.ParticipantDataMigrate; -import org.broadinstitute.dsm.model.elastic.migrationscript.ParticipantMigrate; +import org.broadinstitute.dsm.model.elastic.migrationscript.*; import org.broadinstitute.dsm.util.ParticipantUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -66,19 +62,8 @@ public static void subscribeDSMtasks(String projectId, String subscriptionId) { case ELASTIC_EXPORT: consumer.ack(); ExportToES.ExportPayload exportPayload = new Gson().fromJson(data, ExportToES.ExportPayload.class); - if (exportPayload.isNewExport()) { - String study = exportPayload.getStudy(); - Optional maybeDdpInstanceByInstanceName = - new DDPInstanceDao().getDDPInstanceByInstanceName(study); - maybeDdpInstanceByInstanceName.ifPresent(ddpInstanceDto -> { - String index = ddpInstanceDto.getEsParticipantIndex(); - List exportables = Arrays.asList(new MedicalRecordMigrate(index, study), - new OncHistoryDetailsMigrate(index, study), - new ParticipantDataMigrate(index, study), - new ParticipantMigrate(index, study), - new KitRequestShippingMigrate(index, study)); - exportables.forEach(Exportable::export); - }); + if (exportPayload.isMigration()) { + migrateToES(exportPayload); } else { boolean clearBeforeUpdate = attributesMap.containsKey(CLEAR_BEFORE_UPDATE) && Boolean.parseBoolean(attributesMap.get(CLEAR_BEFORE_UPDATE)); @@ -112,6 +97,23 @@ public static void subscribeDSMtasks(String projectId, String subscriptionId) { } } + private static void migrateToES(ExportToES.ExportPayload exportPayload) { + String study = exportPayload.getStudy(); + Optional maybeDdpInstanceByInstanceName = + new DDPInstanceDao().getDDPInstanceByInstanceName(study); + maybeDdpInstanceByInstanceName.ifPresent(ddpInstanceDto -> { + String index = ddpInstanceDto.getEsParticipantIndex(); + Exportable medicalRecordMigrate = new MedicalRecordMigrate(index, study); + medicalRecordMigrate.export(); +// List exportables = Arrays.asList(new MedicalRecordMigrate(index, study), +// new OncHistoryDetailsMigrate(index, study), +// new ParticipantDataMigrate(index, study), +// new ParticipantMigrate(index, study), +// new KitRequestShippingMigrate(index, study)); +// exportables.forEach(Exportable::export); + }); + } + private static void generateStudyDefaultValues(AckReplyConsumer consumer, Map attributesMap) { String studyGuid = attributesMap.get("studyGuid"); String participantGuid = attributesMap.get("participantGuid"); From a8e51962b8bc4ad6d723a333bbcacc85b0d5ba6c Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 11 Nov 2021 17:16:13 +0400 Subject: [PATCH 110/405] DDP-7117 | minor clean up --- .../broadinstitute/dsm/db/MedicalRecord.java | 1 - .../elastic/migrationscript/BaseMigrator.java | 85 +++---------------- 2 files changed, 12 insertions(+), 74 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java index 18765f86a..b418f3339 100644 --- a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java +++ b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java @@ -441,7 +441,6 @@ public static Map> getMedicalRecords(@NonNull String while (rs.next()) { String ddpParticipantId = rs.getString(DBConstants.DDP_PARTICIPANT_ID); List medicalRecordList = new ArrayList<>(); - if (medicalRecords.keySet().size() > 15) break; if (medicalRecords.containsKey(ddpParticipantId)) { medicalRecordList = medicalRecords.get(ddpParticipantId); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java index 51c2b4e81..38597fed9 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java @@ -1,31 +1,23 @@ package org.broadinstitute.dsm.model.elastic.migrationscript; -import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ForkJoinTask; -import java.util.concurrent.RecursiveAction; -import org.broadinstitute.dsm.db.MedicalRecord; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.Exportable; import org.broadinstitute.dsm.model.elastic.export.generate.Generator; import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchable; -import org.broadinstitute.dsm.pubsub.DSMtasksSubscription; import org.broadinstitute.dsm.statics.ESObjectConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import spark.utils.StringUtils; -public abstract class BaseMigrator extends RecursiveAction implements Exportable, Generator { +public abstract class BaseMigrator implements Exportable, Generator { private static final Logger logger = LoggerFactory.getLogger(BaseMigrator.class); private final ElasticSearchable elasticSearch; - public Map> dataByRealm; protected String object; protected final BulkExportFacade bulkExportFacade; private String primaryId; @@ -44,35 +36,6 @@ public BaseMigrator(String index, String realm, String object, String primaryId, this.aClass = aClass; } - @Override - protected void compute() { - if (dataByRealm.size() > 15) { - ForkJoinTask.invokeAll(createSubtasks()); - } else { - processing(dataByRealm); - } - } - - private void processing(Map> data) { - - - } - - private List createSubtasks() { - List subtasks = new ArrayList<>(); - - int low = 0; - - for (int i = 0; i < ; i++) { - - } - subtasks.add(new CustomRecursiveAction(partOne)); - subtasks.add(new CustomRecursiveAction(partTwo)); - - - return subtasks; - } - private void setPrimaryId() { for(Map map: transformedList) { map.put(Util.ID, map.get(primaryId)); @@ -84,48 +47,24 @@ public Map generate() { return Map.of(ESObjectConstants.DSM, Map.of(object, transformedList)); } - protected void fillBulkRequestWithTransformedMap() { - - if (participantRecords.size() < 15) { - for (Map.Entry> entry: participantRecords.entrySet()) { - String participantId = entry.getKey(); - List recordList = entry.getValue(); - participantId = getParticipantGuid(participantId, index); - if (StringUtils.isBlank(participantId)) continue; - transformedList = Util.transformObjectCollectionToCollectionMap(recordList); - setPrimaryId(); - bulkExportFacade.addDataToRequest(generate(), participantId); - System.err.println(participantId); //FOR TESTING - } - } else { - ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() - 1); - - - executorService.execute(() -> { - for (Map.Entry> entry: participantRecords.entrySet()) { - String participantId = entry.getKey(); - List recordList = entry.getValue(); - participantId = getParticipantGuid(participantId, index); - if (StringUtils.isBlank(participantId)) continue; - transformedList = Util.transformObjectCollectionToCollectionMap(recordList); - setPrimaryId(); - bulkExportFacade.addDataToRequest(generate(), participantId); - System.err.println(participantId); //FOR TESTING - } - }); + protected void fillBulkRequestWithTransformedMap(Map> participantRecords) { + for (Map.Entry> entry: participantRecords.entrySet()) { + String participantId = entry.getKey(); + List recordList = entry.getValue(); + participantId = getParticipantGuid(participantId, index); + if (StringUtils.isBlank(participantId)) continue; + transformedList = Util.transformObjectCollectionToCollectionMap(recordList); + setPrimaryId(); + bulkExportFacade.addDataToRequest(generate(), participantId); + System.err.println(participantId); //FOR TESTING } - } protected abstract Map> getDataByRealm(); @Override public void export() { - dataByRealm = getDataByRealm(); - - fillBulkRequestWithTransformedMap(); - - + fillBulkRequestWithTransformedMap(getDataByRealm()); bulkExportFacade.executeBulkUpsert(); logger.info("finished migrating data to ES."); } From 82fe6ad4056a1d57614858887f212bcfacd7ac37 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 11 Nov 2021 18:26:29 +0400 Subject: [PATCH 111/405] DDP-7117 | fix bug of elastic export because of follow ups --- .../org/broadinstitute/dsm/model/elastic/Util.java | 2 +- .../elastic/migrationscript/BulkExportFacade.java | 4 +++- .../dsm/pubsub/DSMtasksSubscription.java | 14 ++++++-------- .../migrationscript/MedicalRecordMigrateTest.java | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 649189192..cac77eed2 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -108,7 +108,7 @@ public static Map transformObjectToMap(Object obj) { private static Map convertToMap(String fieldName, Object fieldValue) { Map finalResult; switch (fieldName) { - case "followUps": + case "follow_ups": finalResult = Map.of(underscoresToCamelCase(fieldName), new Gson().toJson(fieldValue)); break; case "data": diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BulkExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BulkExportFacade.java index 35d1c5692..96e4a319c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BulkExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BulkExportFacade.java @@ -34,7 +34,9 @@ private UpdateRequest createRequest(Map mapToUpsert, String docId) { public void executeBulkUpsert() { RestHighLevelClient client = ElasticSearchUtil.getClientInstance(); try { - client.bulk(bulkRequest, RequestOptions.DEFAULT); + if (bulkRequest.requests().size() > 0) { + client.bulk(bulkRequest, RequestOptions.DEFAULT); + } } catch (IOException e) { throw new RuntimeException(e); } diff --git a/src/main/java/org/broadinstitute/dsm/pubsub/DSMtasksSubscription.java b/src/main/java/org/broadinstitute/dsm/pubsub/DSMtasksSubscription.java index 5ca34d12e..9ca137845 100644 --- a/src/main/java/org/broadinstitute/dsm/pubsub/DSMtasksSubscription.java +++ b/src/main/java/org/broadinstitute/dsm/pubsub/DSMtasksSubscription.java @@ -103,14 +103,12 @@ private static void migrateToES(ExportToES.ExportPayload exportPayload) { new DDPInstanceDao().getDDPInstanceByInstanceName(study); maybeDdpInstanceByInstanceName.ifPresent(ddpInstanceDto -> { String index = ddpInstanceDto.getEsParticipantIndex(); - Exportable medicalRecordMigrate = new MedicalRecordMigrate(index, study); - medicalRecordMigrate.export(); -// List exportables = Arrays.asList(new MedicalRecordMigrate(index, study), -// new OncHistoryDetailsMigrate(index, study), -// new ParticipantDataMigrate(index, study), -// new ParticipantMigrate(index, study), -// new KitRequestShippingMigrate(index, study)); -// exportables.forEach(Exportable::export); + List exportables = Arrays.asList(new MedicalRecordMigrate(index, study), + new OncHistoryDetailsMigrate(index, study), + new ParticipantDataMigrate(index, study), + new ParticipantMigrate(index, study), + new KitRequestShippingMigrate(index, study)); + exportables.forEach(Exportable::export); }); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java index d22a05a00..e6906af21 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java @@ -45,7 +45,7 @@ public void generateSource() { @Test public void export() { - BaseMigrator migrator = new MedicalRecordMigrate("participants_structured.cmi.cmi-brain","brain"); + BaseMigrator migrator = new MedicalRecordMigrate("participants_structured.atcp.atcp","atcp"); migrator.export(); } From e3c265017c83617ffe1a6b1fc7027e892d8da7b0 Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 11 Nov 2021 19:27:45 +0400 Subject: [PATCH 112/405] DDP-7117 | create BaseCollectionMigrator for ES data which has one-to-many relationship --- .../BaseCollectionMigrator.java | 25 +++++++++++++++++ .../elastic/migrationscript/BaseMigrator.java | 23 ++++++---------- .../KitRequestShippingMigrate.java | 9 ++----- .../migrationscript/MedicalRecordMigrate.java | 4 +-- .../OncHistoryDetailsMigrate.java | 5 ++-- .../ParticipantDataMigrate.java | 4 +-- .../migrationscript/ParticipantMigrate.java | 27 ++++++++++--------- .../KitRequestShippingMigrateTest.java | 2 +- .../MedicalRecordMigrateTest.java | 2 +- .../OncHistoryDetailsMigrateTest.java | 2 +- .../ParticipantDataMigrateTest.java | 2 +- .../ParticipantMigrateTest.java | 2 +- 12 files changed, 60 insertions(+), 47 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseCollectionMigrator.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseCollectionMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseCollectionMigrator.java new file mode 100644 index 000000000..f7fbacc5b --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseCollectionMigrator.java @@ -0,0 +1,25 @@ +package org.broadinstitute.dsm.model.elastic.migrationscript; + +import org.broadinstitute.dsm.model.elastic.Util; + +import java.util.List; +import java.util.Map; + +public abstract class BaseCollectionMigrator extends BaseMigrator { + + public BaseCollectionMigrator(String index, String realm, String object, String primaryId, Class aClass) { + super(index, realm, object, primaryId, aClass); + } + + @Override + protected void transformObject(Object object) { + transformedList = Util.transformObjectCollectionToCollectionMap((List) object); + setPrimaryId(); + } + + private void setPrimaryId() { + for(Map map: transformedList) { + map.put(Util.ID, map.get(primaryId)); + } + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java index 38597fed9..d5169cfd1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java @@ -20,11 +20,11 @@ public abstract class BaseMigrator implements Exportable, Generator { private final ElasticSearchable elasticSearch; protected String object; protected final BulkExportFacade bulkExportFacade; - private String primaryId; + protected String primaryId; protected final String realm; protected final String index; private Class aClass; - private List> transformedList; + protected List> transformedList; public BaseMigrator(String index, String realm, String object, String primaryId, Class aClass) { bulkExportFacade = new BulkExportFacade(index); @@ -36,31 +36,24 @@ public BaseMigrator(String index, String realm, String object, String primaryId, this.aClass = aClass; } - private void setPrimaryId() { - for(Map map: transformedList) { - map.put(Util.ID, map.get(primaryId)); - } - } - @Override public Map generate() { return Map.of(ESObjectConstants.DSM, Map.of(object, transformedList)); } - protected void fillBulkRequestWithTransformedMap(Map> participantRecords) { - for (Map.Entry> entry: participantRecords.entrySet()) { + protected void fillBulkRequestWithTransformedMap(Map participantRecords) { + for (Map.Entry entry: participantRecords.entrySet()) { String participantId = entry.getKey(); - List recordList = entry.getValue(); participantId = getParticipantGuid(participantId, index); if (StringUtils.isBlank(participantId)) continue; - transformedList = Util.transformObjectCollectionToCollectionMap(recordList); - setPrimaryId(); + transformObject(entry.getValue()); bulkExportFacade.addDataToRequest(generate(), participantId); - System.err.println(participantId); //FOR TESTING } } - protected abstract Map> getDataByRealm(); + protected abstract void transformObject(Object object); + + protected abstract Map getDataByRealm(); @Override public void export() { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrate.java index ac581658a..2394ca69f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrate.java @@ -1,24 +1,19 @@ package org.broadinstitute.dsm.model.elastic.migrationscript; -import org.broadinstitute.dsm.db.DDPInstance; import org.broadinstitute.dsm.db.KitRequestShipping; import org.broadinstitute.dsm.statics.ESObjectConstants; -import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.BOOLEAN_MAPPING; -import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.DATE_MAPPING; -import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.TEXT_KEYWORD_MAPPING; - import java.util.List; import java.util.Map; -public class KitRequestShippingMigrate extends BaseMigrator { +public class KitRequestShippingMigrate extends BaseCollectionMigrator { public KitRequestShippingMigrate(String index, String realm) { super(index, realm, ESObjectConstants.KIT_REQUEST_SHIPPING, "dsmKitRequestId", KitRequestShipping.class); } @Override - protected Map> getDataByRealm() { + protected Map getDataByRealm() { Map> kitRequests = KitRequestShipping.getAllKitRequestsByRealm(realm, null, null, true); return (Map) kitRequests; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java index 9a38a1c88..d087ee35d 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java @@ -5,14 +5,14 @@ import org.broadinstitute.dsm.db.MedicalRecord; import org.broadinstitute.dsm.statics.ESObjectConstants; -public class MedicalRecordMigrate extends BaseMigrator { +public class MedicalRecordMigrate extends BaseCollectionMigrator { public MedicalRecordMigrate(String index, String realm) { super(index, realm, ESObjectConstants.MEDICAL_RECORDS, "medicalRecordId", MedicalRecord.class); } @Override - protected Map> getDataByRealm() { + protected Map getDataByRealm() { return (Map) MedicalRecord.getMedicalRecords(realm); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java index aaa8360ec..8ef159907 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java @@ -3,11 +3,10 @@ import org.broadinstitute.dsm.db.OncHistoryDetail; import org.broadinstitute.dsm.statics.ESObjectConstants; -import java.util.List; import java.util.Map; -public class OncHistoryDetailsMigrate extends BaseMigrator { +public class OncHistoryDetailsMigrate extends BaseCollectionMigrator { public OncHistoryDetailsMigrate(String index, String realm) { @@ -15,7 +14,7 @@ public OncHistoryDetailsMigrate(String index, String realm) { } @Override - protected Map> getDataByRealm() { + protected Map getDataByRealm() { return (Map) OncHistoryDetail.getOncHistoryDetails(realm); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java index 87fe75d31..fc01f44c6 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java @@ -15,7 +15,7 @@ import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; import org.broadinstitute.dsm.statics.ESObjectConstants; -public class ParticipantDataMigrate extends BaseMigrator { +public class ParticipantDataMigrate extends BaseCollectionMigrator { public static final String DATA_WITH_ALIAS = "d.data"; private ParticipantDataDao participantDataDao; @@ -26,7 +26,7 @@ public ParticipantDataMigrate(String index, String realm) { } @Override - protected Map> getDataByRealm() { + protected Map getDataByRealm() { Map> participantDataByRealm = participantDataDao.getParticipantDataByRealm(realm); return (Map) participantDataByRealm; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrate.java index b6196fbea..bcb862025 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrate.java @@ -7,10 +7,12 @@ import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchable; import org.broadinstitute.dsm.statics.ESObjectConstants; +import spark.utils.StringUtils; +import java.util.List; import java.util.Map; -public class ParticipantMigrate implements Exportable, Generator { +public class ParticipantMigrate extends BaseMigrator implements Exportable, Generator { private String index; private String realm; @@ -18,26 +20,25 @@ public class ParticipantMigrate implements Exportable, Generator { private Map transformedObject; public ParticipantMigrate(String index, String realm) { + super(index, realm, "participant", "participantId", Participant.class); this.index = index; this.realm = realm; this.bulkExportFacade = new BulkExportFacade(index); } @Override - public void export() { - Map participants = Participant.getParticipants(realm); - for (Map.Entry entry: participants.entrySet()) { - Participant participant = entry.getValue(); - String participantId = entry.getKey(); - transformedObject = Util.transformObjectToMap(participant); - bulkExportFacade.addDataToRequest(generate(), getParticipantGuid(participantId, index)); - } - - bulkExportFacade.executeBulkUpsert(); + public Map generate() { + return Map.of(ESObjectConstants.DSM, Map.of("participant", transformedObject)); } @Override - public Map generate() { - return Map.of(ESObjectConstants.DSM, Map.of("participant", transformedObject)); + protected void transformObject(Object object) { + transformedObject = Util.transformObjectToMap(object); // + } + + @Override + protected Map getDataByRealm() { + return (Map) Participant.getParticipants(realm); + } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrateTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrateTest.java index ff11cd7ad..3804a950e 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrateTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrateTest.java @@ -13,7 +13,7 @@ public void setUp() { @Test public void export() { - KitRequestShippingMigrate kitRequestShippingMigrate = new KitRequestShippingMigrate("participants_structured.cmi.cmi-brain", "brain"); + KitRequestShippingMigrate kitRequestShippingMigrate = new KitRequestShippingMigrate("participants_structured.rgp.rgp", "rgp"); kitRequestShippingMigrate.export(); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java index e6906af21..b32782a51 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java @@ -45,7 +45,7 @@ public void generateSource() { @Test public void export() { - BaseMigrator migrator = new MedicalRecordMigrate("participants_structured.atcp.atcp","atcp"); + BaseMigrator migrator = new MedicalRecordMigrate("participants_structured.rgp.rgp","rgp"); migrator.export(); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrateTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrateTest.java index 32cb08664..23a74c416 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrateTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrateTest.java @@ -15,7 +15,7 @@ public void setUp() throws Exception { @Test public void export() { - OncHistoryDetailsMigrate oncHistoryDetailsMigrate = new OncHistoryDetailsMigrate("participants_structured.cmi.cmi-brain","brain"); + OncHistoryDetailsMigrate oncHistoryDetailsMigrate = new OncHistoryDetailsMigrate("participants_structured.rgp.rgp","rgp"); oncHistoryDetailsMigrate.export(); } } \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrateTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrateTest.java index 068b51435..174f29fd6 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrateTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrateTest.java @@ -13,7 +13,7 @@ public static void setUp() { @Test public void export() { - BaseMigrator participantMigrator = new ParticipantDataMigrate("participants_structured.atcp.atcp", "atcp"); + BaseMigrator participantMigrator = new ParticipantDataMigrate("participants_structured.rgp.rgp", "rgp"); participantMigrator.export(); } } \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrateTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrateTest.java index ee0ca725a..629c1f98f 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrateTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrateTest.java @@ -15,7 +15,7 @@ public void setUp() { @Test public void export() { - ParticipantMigrate participantMigrate = new ParticipantMigrate("participants_structured.cmi.cmi-brain", "brain"); + ParticipantMigrate participantMigrate = new ParticipantMigrate("participants_structured.rgp.rgp", "rgp"); participantMigrate.export(); } } \ No newline at end of file From e80bfcd00ef718d8f828e12f23f051281a149caf Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 11 Nov 2021 19:50:28 +0400 Subject: [PATCH 113/405] DDP-7117 | refactor base migrator --- .../BaseCollectionMigrator.java | 14 ++++++++++++-- .../elastic/migrationscript/BaseMigrator.java | 19 +------------------ .../KitRequestShippingMigrate.java | 2 +- .../migrationscript/MedicalRecordMigrate.java | 2 +- .../OncHistoryDetailsMigrate.java | 2 +- .../ParticipantDataMigrate.java | 8 ++++---- .../migrationscript/ParticipantMigrate.java | 12 ++++-------- .../ParticipantMigrateTest.java | 2 +- 8 files changed, 25 insertions(+), 36 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseCollectionMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseCollectionMigrator.java index f7fbacc5b..673580860 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseCollectionMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseCollectionMigrator.java @@ -1,14 +1,24 @@ package org.broadinstitute.dsm.model.elastic.migrationscript; import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.statics.ESObjectConstants; import java.util.List; import java.util.Map; public abstract class BaseCollectionMigrator extends BaseMigrator { - public BaseCollectionMigrator(String index, String realm, String object, String primaryId, Class aClass) { - super(index, realm, object, primaryId, aClass); + protected String primaryId; + protected List> transformedList; + + public BaseCollectionMigrator(String index, String realm, String object, String primaryId) { + super(index, realm, object); + this.primaryId = primaryId; + } + + @Override + public Map generate() { + return Map.of(ESObjectConstants.DSM, Map.of(object, transformedList)); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java index d5169cfd1..5a9683b79 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java @@ -1,14 +1,9 @@ package org.broadinstitute.dsm.model.elastic.migrationscript; -import java.util.List; import java.util.Map; -import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.Exportable; import org.broadinstitute.dsm.model.elastic.export.generate.Generator; -import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; -import org.broadinstitute.dsm.model.elastic.search.ElasticSearchable; -import org.broadinstitute.dsm.statics.ESObjectConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import spark.utils.StringUtils; @@ -17,28 +12,16 @@ public abstract class BaseMigrator implements Exportable, Generator { private static final Logger logger = LoggerFactory.getLogger(BaseMigrator.class); - private final ElasticSearchable elasticSearch; protected String object; protected final BulkExportFacade bulkExportFacade; - protected String primaryId; protected final String realm; protected final String index; - private Class aClass; - protected List> transformedList; - public BaseMigrator(String index, String realm, String object, String primaryId, Class aClass) { + public BaseMigrator(String index, String realm, String object) { bulkExportFacade = new BulkExportFacade(index); - this.primaryId = primaryId; - this.elasticSearch = new ElasticSearch(); this.realm = realm; this.index = index; this.object = object; - this.aClass = aClass; - } - - @Override - public Map generate() { - return Map.of(ESObjectConstants.DSM, Map.of(object, transformedList)); } protected void fillBulkRequestWithTransformedMap(Map participantRecords) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrate.java index 2394ca69f..0cd1eb6e5 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrate.java @@ -9,7 +9,7 @@ public class KitRequestShippingMigrate extends BaseCollectionMigrator { public KitRequestShippingMigrate(String index, String realm) { - super(index, realm, ESObjectConstants.KIT_REQUEST_SHIPPING, "dsmKitRequestId", KitRequestShipping.class); + super(index, realm, ESObjectConstants.KIT_REQUEST_SHIPPING, "dsmKitRequestId"); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java index d087ee35d..5a0a97fc0 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java @@ -8,7 +8,7 @@ public class MedicalRecordMigrate extends BaseCollectionMigrator { public MedicalRecordMigrate(String index, String realm) { - super(index, realm, ESObjectConstants.MEDICAL_RECORDS, "medicalRecordId", MedicalRecord.class); + super(index, realm, ESObjectConstants.MEDICAL_RECORDS, "medicalRecordId"); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java index 8ef159907..c6147c440 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java @@ -10,7 +10,7 @@ public class OncHistoryDetailsMigrate extends BaseCollectionMigrator { public OncHistoryDetailsMigrate(String index, String realm) { - super(index, realm, ESObjectConstants.ONC_HISTORY_DETAIL_RECORDS, "oncHistoryDetailId", OncHistoryDetail.class); + super(index, realm, ESObjectConstants.ONC_HISTORY_DETAIL_RECORDS, "oncHistoryDetailId"); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java index fc01f44c6..1f674a0c3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java @@ -21,7 +21,7 @@ public class ParticipantDataMigrate extends BaseCollectionMigrator { private ParticipantDataDao participantDataDao; public ParticipantDataMigrate(String index, String realm) { - super(index, realm, ESObjectConstants.PARTICIPANT_DATA, "participantDataId", ParticipantDataDto.class); + super(index, realm, ESObjectConstants.PARTICIPANT_DATA, "participantDataId"); participantDataDao = new ParticipantDataDao(); } @@ -39,13 +39,13 @@ public void export() { private void exportMapping() { Map mapping = new HashMap<>(); - Map> dataByRealm = getDataByRealm(); + Map dataByRealm = getDataByRealm(); BaseParser typeParser = new TypeParser(); try { Field dataField = ParticipantDataDto.class.getDeclaredField(ParticipantDataDao.DATA); dataField.setAccessible(true); - for (Map.Entry> entry : dataByRealm.entrySet()) { - List participantDatas = entry.getValue(); + for (Map.Entry entry : dataByRealm.entrySet()) { + List participantDatas = (List) entry.getValue(); for (Object jsonData : participantDatas) { NameValue nameValue = new NameValue(DATA_WITH_ALIAS, dataField.get(jsonData)); GeneratorPayload generatorPayload = new GeneratorPayload(nameValue); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrate.java index bcb862025..3af7754dd 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrate.java @@ -1,16 +1,12 @@ package org.broadinstitute.dsm.model.elastic.migrationscript; +import java.util.Map; + import org.broadinstitute.dsm.db.Participant; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.Exportable; import org.broadinstitute.dsm.model.elastic.export.generate.Generator; -import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; -import org.broadinstitute.dsm.model.elastic.search.ElasticSearchable; import org.broadinstitute.dsm.statics.ESObjectConstants; -import spark.utils.StringUtils; - -import java.util.List; -import java.util.Map; public class ParticipantMigrate extends BaseMigrator implements Exportable, Generator { @@ -20,7 +16,7 @@ public class ParticipantMigrate extends BaseMigrator implements Exportable, Gene private Map transformedObject; public ParticipantMigrate(String index, String realm) { - super(index, realm, "participant", "participantId", Participant.class); + super(index, realm, "participant"); this.index = index; this.realm = realm; this.bulkExportFacade = new BulkExportFacade(index); @@ -33,7 +29,7 @@ public Map generate() { @Override protected void transformObject(Object object) { - transformedObject = Util.transformObjectToMap(object); // + transformedObject = Util.transformObjectToMap(object); } @Override diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrateTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrateTest.java index 629c1f98f..ee0ca725a 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrateTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrateTest.java @@ -15,7 +15,7 @@ public void setUp() { @Test public void export() { - ParticipantMigrate participantMigrate = new ParticipantMigrate("participants_structured.rgp.rgp", "rgp"); + ParticipantMigrate participantMigrate = new ParticipantMigrate("participants_structured.cmi.cmi-brain", "brain"); participantMigrate.export(); } } \ No newline at end of file From 95d7f687a83c0513fda9bc53da3584a7a95114d1 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 12 Nov 2021 18:27:20 +0400 Subject: [PATCH 114/405] DDP-7117 | remove unnecessary fields --- .../java/org/broadinstitute/dsm/model/elastic/ESDsm.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java index c0ddfab09..bcdb0d312 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java @@ -43,15 +43,9 @@ public class ESDsm { @SerializedName(ESObjectConstants.ONC_HISTORY_DETAIL_RECORDS) List> oncHistoryDetailRecords; - @SerializedName(ESObjectConstants.ONC_HISTORY) - Map oncHistory; - @SerializedName(ESObjectConstants.PARTICIPANT_DATA) List> participantData; - @SerializedName(ESObjectConstants.PARTICIPANT_RECORD) - Map participantRecord; - @SerializedName(ESObjectConstants.PARTICIPANT) Map participant; From 60da40a9561767aa07d70da51bfd0f3253cbf1b8 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 12 Nov 2021 19:34:12 +0400 Subject: [PATCH 115/405] DDP-7117 | migrate final clean up --- .../BaseCollectionMigrator.java | 8 +-- .../BaseMigrator.java | 10 ++-- .../BulkExportFacade.java | 18 ++++--- .../KitRequestShippingMigrator.java} | 12 ++--- .../MedicalRecordMigrator.java} | 8 +-- .../OncHistoryDetailsMigrator.java} | 10 ++-- .../ParticipantDataMigrator.java} | 10 ++-- .../ParticipantMigrator.java} | 12 ++--- .../dsm/pubsub/DSMtasksSubscription.java | 12 ++--- .../KitRequestShippingMigrateTest.java | 20 ------- .../MedicalRecordMigrateTest.java | 52 ------------------- .../OncHistoryDetailsMigrateTest.java | 21 -------- .../ParticipantDataMigrateTest.java | 19 ------- .../ParticipantMigrateTest.java | 21 -------- 14 files changed, 55 insertions(+), 178 deletions(-) rename src/main/java/org/broadinstitute/dsm/model/elastic/{migrationscript => migration}/BaseCollectionMigrator.java (94%) rename src/main/java/org/broadinstitute/dsm/model/elastic/{migrationscript => migration}/BaseMigrator.java (75%) rename src/main/java/org/broadinstitute/dsm/model/elastic/{migrationscript => migration}/BulkExportFacade.java (83%) rename src/main/java/org/broadinstitute/dsm/model/elastic/{migrationscript/KitRequestShippingMigrate.java => migration/KitRequestShippingMigrator.java} (71%) rename src/main/java/org/broadinstitute/dsm/model/elastic/{migrationscript/MedicalRecordMigrate.java => migration/MedicalRecordMigrator.java} (61%) rename src/main/java/org/broadinstitute/dsm/model/elastic/{migrationscript/OncHistoryDetailsMigrate.java => migration/OncHistoryDetailsMigrator.java} (65%) rename src/main/java/org/broadinstitute/dsm/model/elastic/{migrationscript/ParticipantDataMigrate.java => migration/ParticipantDataMigrator.java} (90%) rename src/main/java/org/broadinstitute/dsm/model/elastic/{migrationscript/ParticipantMigrate.java => migration/ParticipantMigrator.java} (68%) delete mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrateTest.java delete mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java delete mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrateTest.java delete mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrateTest.java delete mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrateTest.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseCollectionMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java similarity index 94% rename from src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseCollectionMigrator.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java index 673580860..3db138d51 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseCollectionMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java @@ -1,11 +1,11 @@ -package org.broadinstitute.dsm.model.elastic.migrationscript; - -import org.broadinstitute.dsm.model.elastic.Util; -import org.broadinstitute.dsm.statics.ESObjectConstants; +package org.broadinstitute.dsm.model.elastic.migration; import java.util.List; import java.util.Map; +import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.statics.ESObjectConstants; + public abstract class BaseCollectionMigrator extends BaseMigrator { protected String primaryId; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseMigrator.java similarity index 75% rename from src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseMigrator.java index 5a9683b79..5548cfc96 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BaseMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseMigrator.java @@ -1,4 +1,4 @@ -package org.broadinstitute.dsm.model.elastic.migrationscript; +package org.broadinstitute.dsm.model.elastic.migration; import java.util.Map; @@ -25,13 +25,17 @@ public BaseMigrator(String index, String realm, String object) { } protected void fillBulkRequestWithTransformedMap(Map participantRecords) { + logger.info("filling bulk request with participants and their details"); for (Map.Entry entry: participantRecords.entrySet()) { String participantId = entry.getKey(); participantId = getParticipantGuid(participantId, index); if (StringUtils.isBlank(participantId)) continue; - transformObject(entry.getValue()); - bulkExportFacade.addDataToRequest(generate(), participantId); + Object participantDetails = entry.getValue(); + transformObject(participantDetails); + Map finalMapToUpsert = generate(); + bulkExportFacade.addDataToRequest(finalMapToUpsert, participantId); } + logger.info("successfully filled bulk request with participants and their details"); } protected abstract void transformObject(Object object); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BulkExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BulkExportFacade.java similarity index 83% rename from src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BulkExportFacade.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/migration/BulkExportFacade.java index 96e4a319c..0c0e0cb17 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/BulkExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BulkExportFacade.java @@ -1,18 +1,22 @@ -package org.broadinstitute.dsm.model.elastic.migrationscript; +package org.broadinstitute.dsm.model.elastic.migration; + +import static org.broadinstitute.dsm.model.elastic.Util.DOC; + +import java.io.IOException; +import java.util.Map; import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; - -import java.io.IOException; -import java.util.Map; - -import static org.broadinstitute.dsm.model.elastic.Util.DOC; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class BulkExportFacade { + private static final Logger logger = LoggerFactory.getLogger(BulkExportFacade.class); + private String index; private BulkRequest bulkRequest; @@ -34,12 +38,14 @@ private UpdateRequest createRequest(Map mapToUpsert, String docId) { public void executeBulkUpsert() { RestHighLevelClient client = ElasticSearchUtil.getClientInstance(); try { + logger.info("attempting to upsert participants"); if (bulkRequest.requests().size() > 0) { client.bulk(bulkRequest, RequestOptions.DEFAULT); } } catch (IOException e) { throw new RuntimeException(e); } + } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/KitRequestShippingMigrator.java similarity index 71% rename from src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrate.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/migration/KitRequestShippingMigrator.java index 0cd1eb6e5..bba3f8fa5 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/KitRequestShippingMigrator.java @@ -1,14 +1,14 @@ -package org.broadinstitute.dsm.model.elastic.migrationscript; - -import org.broadinstitute.dsm.db.KitRequestShipping; -import org.broadinstitute.dsm.statics.ESObjectConstants; +package org.broadinstitute.dsm.model.elastic.migration; import java.util.List; import java.util.Map; -public class KitRequestShippingMigrate extends BaseCollectionMigrator { +import org.broadinstitute.dsm.db.KitRequestShipping; +import org.broadinstitute.dsm.statics.ESObjectConstants; + +public class KitRequestShippingMigrator extends BaseCollectionMigrator { - public KitRequestShippingMigrate(String index, String realm) { + public KitRequestShippingMigrator(String index, String realm) { super(index, realm, ESObjectConstants.KIT_REQUEST_SHIPPING, "dsmKitRequestId"); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigrator.java similarity index 61% rename from src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigrator.java index 5a0a97fc0..2478d3e68 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigrator.java @@ -1,13 +1,13 @@ -package org.broadinstitute.dsm.model.elastic.migrationscript; +package org.broadinstitute.dsm.model.elastic.migration; -import java.util.*; +import java.util.Map; import org.broadinstitute.dsm.db.MedicalRecord; import org.broadinstitute.dsm.statics.ESObjectConstants; -public class MedicalRecordMigrate extends BaseCollectionMigrator { +public class MedicalRecordMigrator extends BaseCollectionMigrator { - public MedicalRecordMigrate(String index, String realm) { + public MedicalRecordMigrator(String index, String realm) { super(index, realm, ESObjectConstants.MEDICAL_RECORDS, "medicalRecordId"); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java similarity index 65% rename from src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java index c6147c440..87141bc9e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java @@ -1,15 +1,15 @@ -package org.broadinstitute.dsm.model.elastic.migrationscript; +package org.broadinstitute.dsm.model.elastic.migration; + +import java.util.Map; import org.broadinstitute.dsm.db.OncHistoryDetail; import org.broadinstitute.dsm.statics.ESObjectConstants; -import java.util.Map; - -public class OncHistoryDetailsMigrate extends BaseCollectionMigrator { +public class OncHistoryDetailsMigrator extends BaseCollectionMigrator { - public OncHistoryDetailsMigrate(String index, String realm) { + public OncHistoryDetailsMigrator(String index, String realm) { super(index, realm, ESObjectConstants.ONC_HISTORY_DETAIL_RECORDS, "oncHistoryDetailId"); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java similarity index 90% rename from src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java index 1f674a0c3..4d310395d 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java @@ -1,8 +1,10 @@ -package org.broadinstitute.dsm.model.elastic.migrationscript; +package org.broadinstitute.dsm.model.elastic.migration; import java.lang.reflect.Field; -import java.util.*; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; @@ -15,12 +17,12 @@ import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; import org.broadinstitute.dsm.statics.ESObjectConstants; -public class ParticipantDataMigrate extends BaseCollectionMigrator { +public class ParticipantDataMigrator extends BaseCollectionMigrator { public static final String DATA_WITH_ALIAS = "d.data"; private ParticipantDataDao participantDataDao; - public ParticipantDataMigrate(String index, String realm) { + public ParticipantDataMigrator(String index, String realm) { super(index, realm, ESObjectConstants.PARTICIPANT_DATA, "participantDataId"); participantDataDao = new ParticipantDataDao(); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrate.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantMigrator.java similarity index 68% rename from src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrate.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantMigrator.java index 3af7754dd..9fa14e974 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrate.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantMigrator.java @@ -1,4 +1,4 @@ -package org.broadinstitute.dsm.model.elastic.migrationscript; +package org.broadinstitute.dsm.model.elastic.migration; import java.util.Map; @@ -8,23 +8,21 @@ import org.broadinstitute.dsm.model.elastic.export.generate.Generator; import org.broadinstitute.dsm.statics.ESObjectConstants; -public class ParticipantMigrate extends BaseMigrator implements Exportable, Generator { +public class ParticipantMigrator extends BaseMigrator implements Exportable, Generator { - private String index; private String realm; protected final BulkExportFacade bulkExportFacade; private Map transformedObject; - public ParticipantMigrate(String index, String realm) { - super(index, realm, "participant"); - this.index = index; + public ParticipantMigrator(String index, String realm) { + super(index, realm, ESObjectConstants.PARTICIPANT); this.realm = realm; this.bulkExportFacade = new BulkExportFacade(index); } @Override public Map generate() { - return Map.of(ESObjectConstants.DSM, Map.of("participant", transformedObject)); + return Map.of(ESObjectConstants.DSM, Map.of(object, transformedObject)); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/pubsub/DSMtasksSubscription.java b/src/main/java/org/broadinstitute/dsm/pubsub/DSMtasksSubscription.java index 9ca137845..453a6d76f 100644 --- a/src/main/java/org/broadinstitute/dsm/pubsub/DSMtasksSubscription.java +++ b/src/main/java/org/broadinstitute/dsm/pubsub/DSMtasksSubscription.java @@ -16,7 +16,7 @@ import org.broadinstitute.dsm.model.defaultvalues.Defaultable; import org.broadinstitute.dsm.model.defaultvalues.DefaultableMaker; import org.broadinstitute.dsm.model.elastic.export.Exportable; -import org.broadinstitute.dsm.model.elastic.migrationscript.*; +import org.broadinstitute.dsm.model.elastic.migration.*; import org.broadinstitute.dsm.util.ParticipantUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -103,11 +103,11 @@ private static void migrateToES(ExportToES.ExportPayload exportPayload) { new DDPInstanceDao().getDDPInstanceByInstanceName(study); maybeDdpInstanceByInstanceName.ifPresent(ddpInstanceDto -> { String index = ddpInstanceDto.getEsParticipantIndex(); - List exportables = Arrays.asList(new MedicalRecordMigrate(index, study), - new OncHistoryDetailsMigrate(index, study), - new ParticipantDataMigrate(index, study), - new ParticipantMigrate(index, study), - new KitRequestShippingMigrate(index, study)); + List exportables = Arrays.asList(new MedicalRecordMigrator(index, study), + new OncHistoryDetailsMigrator(index, study), + new ParticipantDataMigrator(index, study), + new ParticipantMigrator(index, study), + new KitRequestShippingMigrator(index, study)); exportables.forEach(Exportable::export); }); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrateTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrateTest.java deleted file mode 100644 index 3804a950e..000000000 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/KitRequestShippingMigrateTest.java +++ /dev/null @@ -1,20 +0,0 @@ -package org.broadinstitute.dsm.model.elastic.migrationscript; - -import org.broadinstitute.dsm.TestHelper; -import org.junit.Before; -import org.junit.Test; - -public class KitRequestShippingMigrateTest { - - @Before - public void setUp() { - TestHelper.setupDB(); - } - - @Test - public void export() { - KitRequestShippingMigrate kitRequestShippingMigrate = new KitRequestShippingMigrate("participants_structured.rgp.rgp", "rgp"); - kitRequestShippingMigrate.export(); - } - -} \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java deleted file mode 100644 index b32782a51..000000000 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/MedicalRecordMigrateTest.java +++ /dev/null @@ -1,52 +0,0 @@ -package org.broadinstitute.dsm.model.elastic.migrationscript; - -import static org.junit.Assert.*; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -import org.broadinstitute.dsm.TestHelper; -import org.broadinstitute.dsm.db.MedicalRecord; -import org.broadinstitute.dsm.model.elastic.Util; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - -public class MedicalRecordMigrateTest { - - @BeforeClass - public static void setUp() { - TestHelper.setupDB(); - } - - @Test - public void transformMedicalRecordToMap() { - List medicalRecords = Arrays.asList(new MedicalRecord("1", "2", "3", "TYPE")); - List> listOfMaps = Util.transformObjectCollectionToCollectionMap(medicalRecords); - Map stringObjectMap = listOfMaps.get(0); - Assert.assertEquals("1", stringObjectMap.get("medicalRecordId")); - Assert.assertEquals("2", stringObjectMap.get("institutionId")); - Assert.assertEquals("3", stringObjectMap.get("ddpInstitutionId")); - Assert.assertEquals("TYPE", stringObjectMap.get("type")); - } - - @Test - public void generateSource() { - List medicalRecords = Arrays.asList(new MedicalRecord("1", "2", "3", "TYPE")); - List> listOfMaps = Util.transformObjectCollectionToCollectionMap(medicalRecords); - MedicalRecordMigrate migrator = new MedicalRecordMigrate("", ""); - Map resultMap = migrator.generate(); - Map dsm = (Map)resultMap.get("dsm"); - List> medicalRecords1 = (List>) dsm.get("medicalRecords"); - Object medicalRecordsId = medicalRecords1.get(0).get("medicalRecordId"); - assertEquals("1", medicalRecordsId); - } - - @Test - public void export() { - BaseMigrator migrator = new MedicalRecordMigrate("participants_structured.rgp.rgp","rgp"); - migrator.export(); - } - -} \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrateTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrateTest.java deleted file mode 100644 index 23a74c416..000000000 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/OncHistoryDetailsMigrateTest.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.broadinstitute.dsm.model.elastic.migrationscript; - -import static org.junit.Assert.*; - -import org.broadinstitute.dsm.TestHelper; -import org.junit.Before; -import org.junit.Test; - -public class OncHistoryDetailsMigrateTest { - - @Before - public void setUp() throws Exception { - TestHelper.setupDB(); - } - - @Test - public void export() { - OncHistoryDetailsMigrate oncHistoryDetailsMigrate = new OncHistoryDetailsMigrate("participants_structured.rgp.rgp","rgp"); - oncHistoryDetailsMigrate.export(); - } -} \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrateTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrateTest.java deleted file mode 100644 index 174f29fd6..000000000 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantDataMigrateTest.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.broadinstitute.dsm.model.elastic.migrationscript; - -import org.broadinstitute.dsm.TestHelper; -import org.junit.BeforeClass; -import org.junit.Test; - -public class ParticipantDataMigrateTest { - - @BeforeClass - public static void setUp() { - TestHelper.setupDB(); - } - - @Test - public void export() { - BaseMigrator participantMigrator = new ParticipantDataMigrate("participants_structured.rgp.rgp", "rgp"); - participantMigrator.export(); - } -} \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrateTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrateTest.java deleted file mode 100644 index ee0ca725a..000000000 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migrationscript/ParticipantMigrateTest.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.broadinstitute.dsm.model.elastic.migrationscript; - -import static org.junit.Assert.*; - -import org.broadinstitute.dsm.TestHelper; -import org.junit.Before; -import org.junit.Test; - -public class ParticipantMigrateTest { - - @Before - public void setUp() { - TestHelper.setupDB(); - } - - @Test - public void export() { - ParticipantMigrate participantMigrate = new ParticipantMigrate("participants_structured.cmi.cmi-brain", "brain"); - participantMigrate.export(); - } -} \ No newline at end of file From 2aa6a0bede278445089a9d2a96319bf35b64bcde Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 15 Nov 2021 11:03:07 +0400 Subject: [PATCH 116/405] DDP-7117 | use Pattern instead of bare String in order to improve performance --- src/main/java/org/broadinstitute/dsm/model/elastic/Util.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index cac77eed2..50924c3e9 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.lang.reflect.Field; import java.util.*; +import java.util.regex.Pattern; import java.util.stream.Collectors; import com.fasterxml.jackson.databind.ObjectMapper; @@ -30,7 +31,7 @@ public class Util { public static final String UNDERSCORE_SEPARATOR = "_"; public static final String DOC = "_doc"; public static final String ID = "id"; - public static final String CAMEL_CASE_REGEX = "(([a-z])+([A-z]))*"; + private static final Pattern CAMEL_CASE_REGEX = Pattern.compile("(([a-z])+([A-z]))*"); public static String getQueryTypeFromId(String id) { String type; @@ -66,7 +67,7 @@ public static String underscoresToCamelCase(String fieldName) { } private static String handleAllUppercase(String word) { - return word.matches(CAMEL_CASE_REGEX) ? word : word.toLowerCase(); + return CAMEL_CASE_REGEX.matcher(word).matches() ? word : word.toLowerCase(); } private static boolean hasNoUnderscores(String[] splittedWords) { From 3358638827baa13832e04e5dc04b217fa025b877 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 15 Nov 2021 17:25:26 +0400 Subject: [PATCH 117/405] DDP-7121 | change anonymous names --- .../dsm/db/OncHistoryDetail.java | 134 +++++++++--------- .../org/broadinstitute/dsm/db/ViewFilter.java | 2 +- .../dsm/model/PDF/TissueCoverPDF.java | 8 +- .../dsm/model/elastic/ESDsm.java | 12 +- .../dsm/route/DashboardRoute.java | 8 +- 5 files changed, 84 insertions(+), 80 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java b/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java index 48f6e4016..e793dbf87 100644 --- a/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java +++ b/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java @@ -83,13 +83,13 @@ public class OncHistoryDetail { private final String medicalRecordId; @ColumnName (DBConstants.DATE_PX) - private String datePX; + private String datePx; @ColumnName (DBConstants.TYPE_PX) - private String typePX; + private String typePx; @ColumnName (DBConstants.LOCATION_PX) - private String locationPX; + private String locationPx; @ColumnName (DBConstants.HISTOLOGY) private String histology; @@ -101,49 +101,49 @@ public class OncHistoryDetail { private String facility; @ColumnName (DBConstants.PHONE) - private String fPhone; + private String phone; @ColumnName (DBConstants.FAX) - private String fFax; + private String fax; @ColumnName (DBConstants.NOTES) - private String oncHisNotes; + private String notes; @ColumnName (DBConstants.REQUEST) private String request; @ColumnName (DBConstants.FAX_SENT) @DbDateConversion(SqlDateConverter.STRING_DAY) - private String tFaxSent; + private String faxSent; @ColumnName (DBConstants.FAX_SENT_BY) - private String tFaxSentBy; + private String faxSentBy; @ColumnName (DBConstants.FAX_CONFIRMED) @DbDateConversion(SqlDateConverter.STRING_DAY) - private String tFaxConfirmed; + private String faxConfirmed; @ColumnName (DBConstants.FAX_SENT_2) @DbDateConversion(SqlDateConverter.STRING_DAY) - private String tFaxSent2; + private String faxSent2; @ColumnName (DBConstants.FAX_SENT_2_BY) - private String tFaxSent2By; + private String faxSent2By; @ColumnName (DBConstants.FAX_CONFIRMED_2) @DbDateConversion(SqlDateConverter.STRING_DAY) - private String tFaxConfirmed2; + private String faxConfirmed2; @ColumnName (DBConstants.FAX_SENT_3) @DbDateConversion(SqlDateConverter.STRING_DAY) - private String tFaxSent3; + private String faxSent3; @ColumnName (DBConstants.FAX_SENT_3_BY) - private String tFaxSent3By; + private String faxSent3By; @ColumnName (DBConstants.FAX_CONFIRMED_3) @DbDateConversion(SqlDateConverter.STRING_DAY) - private String tFaxConfirmed3; + private String faxConfirmed3; @ColumnName (DBConstants.TISSUE_RECEIVED) @DbDateConversion(SqlDateConverter.STRING_DAY) @@ -156,7 +156,7 @@ public class OncHistoryDetail { private String gender; @ColumnName (DBConstants.ADDITIONAL_VALUES) - private String additionalValues; + private String additionalValuesJson; @ColumnName (DBConstants.DESTRUCTION_POLICY) private String destructionPolicy; @@ -167,87 +167,87 @@ public class OncHistoryDetail { private boolean deleted; @ColumnName (DBConstants.UNABLE_OBTAIN_TISSUE) - private boolean unableToObtain; + private boolean unableObtainTissue; private String participantId; private List tissues; - public OncHistoryDetail(String oncHistoryDetailId, String medicalRecordId, String datePX, String typePX, - String locationPX, String histology, String accessionNumber, String facility, String fPhone, - String fFax, String oncHisNotes, String request, String tFaxSent, - String tFaxSentBy, String tFaxConfirmed, - String tFaxSent2, String tFaxSent2By, String tFaxConfirmed2, - String tFaxSent3, String tFaxSent3By, String tFaxConfirmed3, - String tissueReceived, String gender, String additionalValues, - String tissueProblemOption, String destructionPolicy, boolean unableToObtain) { + public OncHistoryDetail(String oncHistoryDetailId, String medicalRecordId, String datePx, String typePx, + String locationPx, String histology, String accessionNumber, String facility, String phone, + String fax, String notes, String request, String faxSent, + String faxSentBy, String faxConfirmed, + String faxSent2, String faxSent2By, String faxConfirmed2, + String faxSent3, String faxSent3By, String faxConfirmed3, + String tissueReceived, String gender, String additionalValuesJson, + String tissueProblemOption, String destructionPolicy, boolean unableObtainTissue) { this.oncHistoryDetailId = oncHistoryDetailId; this.medicalRecordId = medicalRecordId; - this.datePX = datePX; - this.typePX = typePX; - this.locationPX = locationPX; + this.datePx = datePx; + this.typePx = typePx; + this.locationPx = locationPx; this.histology = histology; this.accessionNumber = accessionNumber; this.facility = facility; - this.fPhone = fPhone; - this.fFax = fFax; - this.oncHisNotes = oncHisNotes; + this.phone = phone; + this.fax = fax; + this.notes = notes; this.request = request; - this.tFaxSent = tFaxSent; - this.tFaxSentBy = tFaxSentBy; - this.tFaxConfirmed = tFaxConfirmed; - this.tFaxSent2 = tFaxSent2; - this.tFaxSent2By = tFaxSent2By; - this.tFaxConfirmed2 = tFaxConfirmed2; - this.tFaxSent3 = tFaxSent3; - this.tFaxSent3By = tFaxSent3By; - this.tFaxConfirmed3 = tFaxConfirmed3; + this.faxSent = faxSent; + this.faxSentBy = faxSentBy; + this.faxConfirmed = faxConfirmed; + this.faxSent2 = faxSent2; + this.faxSent2By = faxSent2By; + this.faxConfirmed2 = faxConfirmed2; + this.faxSent3 = faxSent3; + this.faxSent3By = faxSent3By; + this.faxConfirmed3 = faxConfirmed3; this.tissueReceived = tissueReceived; this.gender = gender; - this.additionalValues = additionalValues; + this.additionalValuesJson = additionalValuesJson; this.tissues = new ArrayList<>(); this.tissueProblemOption = tissueProblemOption; this.destructionPolicy = destructionPolicy; - this.unableToObtain = unableToObtain; + this.unableObtainTissue = unableObtainTissue; } - public OncHistoryDetail(String oncHistoryDetailId, String medicalRecordId, String datePX, String typePX, - String locationPX, String histology, String accessionNumber, String facility, String fPhone, - String fFax, String oncHisNotes, String request, String tFaxSent, - String tFaxSentBy, String tFaxConfirmed, - String tFaxSent2, String tFaxSent2By, String tFaxConfirmed2, - String tFaxSent3, String tFaxSent3By, String tFaxConfirmed3, - String tissueReceived, String gender, String additionalValues, List tissues, - String tissueProblemOption, String destructionPolicy, boolean unableToObtain) { + public OncHistoryDetail(String oncHistoryDetailId, String medicalRecordId, String datePx, String typePx, + String locationPx, String histology, String accessionNumber, String facility, String phone, + String fax, String notes, String request, String faxSent, + String faxSentBy, String faxConfirmed, + String faxSent2, String faxSent2By, String faxConfirmed2, + String faxSent3, String faxSent3By, String faxConfirmed3, + String tissueReceived, String gender, String additionalValuesJson, List tissues, + String tissueProblemOption, String destructionPolicy, boolean unableObtainTissue) { this.oncHistoryDetailId = oncHistoryDetailId; this.medicalRecordId = medicalRecordId; - this.datePX = datePX; - this.typePX = typePX; - this.locationPX = locationPX; + this.datePx = datePx; + this.typePx = typePx; + this.locationPx = locationPx; this.histology = histology; this.accessionNumber = accessionNumber; this.facility = facility; - this.fPhone = fPhone; - this.fFax = fFax; - this.oncHisNotes = oncHisNotes; + this.phone = phone; + this.fax = fax; + this.notes = notes; this.request = request; - this.tFaxSent = tFaxSent; - this.tFaxSentBy = tFaxSentBy; - this.tFaxConfirmed = tFaxConfirmed; - this.tFaxSent2 = tFaxSent2; - this.tFaxSent2By = tFaxSent2By; - this.tFaxConfirmed2 = tFaxConfirmed2; - this.tFaxSent3 = tFaxSent3; - this.tFaxSent3By = tFaxSent3By; - this.tFaxConfirmed3 = tFaxConfirmed3; + this.faxSent = faxSent; + this.faxSentBy = faxSentBy; + this.faxConfirmed = faxConfirmed; + this.faxSent2 = faxSent2; + this.faxSent2By = faxSent2By; + this.faxConfirmed2 = faxConfirmed2; + this.faxSent3 = faxSent3; + this.faxSent3By = faxSent3By; + this.faxConfirmed3 = faxConfirmed3; this.tissueReceived = tissueReceived; this.gender = gender; - this.additionalValues = additionalValues; + this.additionalValuesJson = additionalValuesJson; this.tissues = tissues; this.tissueProblemOption = tissueProblemOption; this.destructionPolicy = destructionPolicy; - this.unableToObtain = unableToObtain; + this.unableObtainTissue = unableObtainTissue; } public static OncHistoryDetail getOncHistoryDetail(@NonNull ResultSet rs) throws SQLException { diff --git a/src/main/java/org/broadinstitute/dsm/db/ViewFilter.java b/src/main/java/org/broadinstitute/dsm/db/ViewFilter.java index 2853b76d3..39a34ab3c 100644 --- a/src/main/java/org/broadinstitute/dsm/db/ViewFilter.java +++ b/src/main/java/org/broadinstitute/dsm/db/ViewFilter.java @@ -366,7 +366,7 @@ public static List getDestroyingSamples(String realm) { loop: for (TissueList tissueList : views) { if (tissueList.getOncHistoryDetails() != null) { - String dateString = tissueList.getOncHistoryDetails().getDatePX(); + String dateString = tissueList.getOncHistoryDetails().getDatePx(); int len = dateString.length(); int destructionPolicy = 1000; if (!tissueList.getOncHistoryDetails().getDestructionPolicy().equals("indefinitely")) { diff --git a/src/main/java/org/broadinstitute/dsm/model/PDF/TissueCoverPDF.java b/src/main/java/org/broadinstitute/dsm/model/PDF/TissueCoverPDF.java index 94591f52a..2059419ef 100644 --- a/src/main/java/org/broadinstitute/dsm/model/PDF/TissueCoverPDF.java +++ b/src/main/java/org/broadinstitute/dsm/model/PDF/TissueCoverPDF.java @@ -37,11 +37,11 @@ public Map getValuesForTissueCover(DDPInstance ddpInstance, User // facility information is the same in all of the requests so only need to be set ones! if (i == 0) { valueMap.put(RequestPDFProcessor.FIELD_CONFIRMED_INSTITUTION_NAME, oncHistoryDetail.getFacility()); - valueMap.put(RequestPDFProcessor.FIELD_CONFIRMED_PHONE, oncHistoryDetail.getFPhone()); - valueMap.put(RequestPDFProcessor.FIELD_CONFIRMED_FAX, oncHistoryDetail.getFFax()); + valueMap.put(RequestPDFProcessor.FIELD_CONFIRMED_PHONE, oncHistoryDetail.getPhone()); + valueMap.put(RequestPDFProcessor.FIELD_CONFIRMED_FAX, oncHistoryDetail.getFax()); } - valueMap.put(RequestPDFProcessor.FIELD_DATE_PX + i, oncHistoryDetail.getDatePX()); - valueMap.put(RequestPDFProcessor.FIELD_TYPE_LOCATION + i, oncHistoryDetail.getTypePX()); + valueMap.put(RequestPDFProcessor.FIELD_DATE_PX + i, oncHistoryDetail.getDatePx()); + valueMap.put(RequestPDFProcessor.FIELD_TYPE_LOCATION + i, oncHistoryDetail.getTypePx()); valueMap.put(RequestPDFProcessor.FIELD_ACCESSION_NUMBER + i, oncHistoryDetail.getAccessionNumber()); counter = i; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java index bcdb0d312..a7fbb735b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java @@ -5,6 +5,10 @@ import com.google.gson.annotations.SerializedName; import lombok.Getter; +import org.broadinstitute.dsm.db.MedicalRecord; +import org.broadinstitute.dsm.db.OncHistoryDetail; +import org.broadinstitute.dsm.db.Participant; +import org.broadinstitute.dsm.db.ParticipantData; import org.broadinstitute.dsm.statics.ESObjectConstants; @Getter @@ -38,16 +42,16 @@ public class ESDsm { List> tissueRecords; @SerializedName(ESObjectConstants.MEDICAL_RECORDS) - List> medicalRecords; + List medicalRecords; @SerializedName(ESObjectConstants.ONC_HISTORY_DETAIL_RECORDS) - List> oncHistoryDetailRecords; + List oncHistoryDetailRecords; @SerializedName(ESObjectConstants.PARTICIPANT_DATA) - List> participantData; + List participantData; @SerializedName(ESObjectConstants.PARTICIPANT) - Map participant; + Participant participant; } diff --git a/src/main/java/org/broadinstitute/dsm/route/DashboardRoute.java b/src/main/java/org/broadinstitute/dsm/route/DashboardRoute.java index 37524ef71..cfda12875 100644 --- a/src/main/java/org/broadinstitute/dsm/route/DashboardRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/DashboardRoute.java @@ -412,7 +412,7 @@ private void countOncHistoryData(@NonNull List oncHistoryDetai long start, long end) { for (OncHistoryDetail oncHistoryDetail : oncHistoryDetailList) { - if (oncHistoryDetail.isUnableToObtain()) { + if (oncHistoryDetail.isUnableObtainTissue()) { incrementCounter(dashboardValuesDetailed, "unableToObtainTissue"); foundAtPT.add("unableToObtainTissue"); } @@ -445,7 +445,7 @@ else if (OncHistoryDetail.STATUS_REQUEST.equals(oncHistoryDetail.getRequest())) incrementCounter(dashboardValuesDetailed, "request.request"); foundAtPT.add("request.request"); //count where tissue request is set to request but fax is not sent out yet - if (StringUtils.isBlank(oncHistoryDetail.getTFaxSent())) { + if (StringUtils.isBlank(oncHistoryDetail.getFaxSent())) { incrementCounter(dashboardValuesDetailed, "tFaxNotSent"); foundAtPT.add("tFaxNotSent"); } @@ -488,8 +488,8 @@ else if (OncHistoryDetail.PROBLEM_OTHER.equals(oncHistoryDetail.getTissueProblem foundAtPT.add("tissueProblemOption.other"); } } - countRequestsReceive(dashboardValuesDetailed, dashboardValuesPeriodDetailed, foundAtPT, foundAtPtPeriod, oncHistoryDetail.getTFaxSent3(), - oncHistoryDetail.getTFaxSent2(), oncHistoryDetail.getTFaxSent(), oncHistoryDetail.getTissueReceived(), + countRequestsReceive(dashboardValuesDetailed, dashboardValuesPeriodDetailed, foundAtPT, foundAtPtPeriod, oncHistoryDetail.getFaxSent3(), + oncHistoryDetail.getFaxSent2(), oncHistoryDetail.getFaxSent(), oncHistoryDetail.getTissueReceived(), start, end, null, "tFaxSent", "tissueReceived", false); } } From 8a333231f5726253c5d207e60c439846b15e7d66 Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 16 Nov 2021 16:22:27 +0400 Subject: [PATCH 118/405] DDP-7117 | refactor code based on nested structures in ES data (cherry picked from commit 017bac47b82bc7740e5a7dffa0378f04e38cd1a1) --- .../dsm/db/OncHistoryDetail.java | 1 - .../org/broadinstitute/dsm/db/Tissue.java | 6 +++ .../dsm/model/elastic/Util.java | 24 ++++++++- .../migration/BaseCollectionMigrator.java | 51 ++++++++++++++++++- .../migration/OncHistoryDetailsMigrator.java | 7 ++- .../dsm/model/elastic/UtilTest.java | 18 +++++++ 6 files changed, 101 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java b/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java index e793dbf87..464b02453 100644 --- a/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java +++ b/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java @@ -171,7 +171,6 @@ public class OncHistoryDetail { private String participantId; - private List tissues; public OncHistoryDetail(String oncHistoryDetailId, String medicalRecordId, String datePx, String typePx, diff --git a/src/main/java/org/broadinstitute/dsm/db/Tissue.java b/src/main/java/org/broadinstitute/dsm/db/Tissue.java index 2a5dad3b0..9fae040ed 100644 --- a/src/main/java/org/broadinstitute/dsm/db/Tissue.java +++ b/src/main/java/org/broadinstitute/dsm/db/Tissue.java @@ -37,7 +37,13 @@ public class Tissue { "LEFT JOIN ddp_medical_record as m on (m.institution_id = inst.institution_id AND NOT m.deleted <=> 1) LEFT JOIN ddp_onc_history_detail as oD on (m.medical_record_id = oD.medical_record_id) " + "LEFT JOIN ddp_tissue as t on (t.onc_history_detail_id = oD.onc_history_detail_id) WHERE p.participant_id = ?"; + @TableName ( + name = DBConstants.DDP_TISSUE, + alias = DBConstants.DDP_TISSUE_ALIAS, + primaryKey = DBConstants.TISSUE_ID, + columnPrefix = "") private String tissueId; + private final String oncHistoryDetailId; @ColumnName (DBConstants.NOTES) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 50924c3e9..e836381df 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -106,7 +106,7 @@ public static Map transformObjectToMap(Object obj) { return map; } - private static Map convertToMap(String fieldName, Object fieldValue) { + static Map convertToMap(String fieldName, Object fieldValue) { Map finalResult; switch (fieldName) { case "follow_ups": @@ -124,6 +124,28 @@ private static Map convertToMap(String fieldName, Object fieldVa finalResult = transformedMap; break; default: + if (fieldValue instanceof List) { + List fieldValues = (List) fieldValue; + List> transformedFieldValues = new ArrayList<>(); + for(Object obj: fieldValues) { + Class inside = obj.getClass(); + Field[] declaredFields = inside.getDeclaredFields(); + for (Field declaredField : declaredFields) { + declaredField.setAccessible(true); + ColumnName annotation = declaredField.getAnnotation(ColumnName.class); + if (annotation != null) { + String innerFieldName = underscoresToCamelCase(annotation.value()); + try { + if (declaredField.get(obj) != null ) + transformedFieldValues.add(Map.of(innerFieldName, declaredField.get(obj))); + } catch (IllegalAccessException e) { + e.printStackTrace(); + } + } + } + } + fieldValue = transformedFieldValues; + } finalResult = Map.of(underscoresToCamelCase(fieldName), fieldValue); break; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java index 3db138d51..a30bd3db1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java @@ -1,8 +1,11 @@ package org.broadinstitute.dsm.model.elastic.migration; -import java.util.List; -import java.util.Map; +import java.lang.reflect.Field; +import java.util.*; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; +import org.broadinstitute.dsm.db.structure.TableName; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.statics.ESObjectConstants; @@ -10,10 +13,12 @@ public abstract class BaseCollectionMigrator extends BaseMigrator { protected String primaryId; protected List> transformedList; + protected Set primaryKeys; public BaseCollectionMigrator(String index, String realm, String object, String primaryId) { super(index, realm, object); this.primaryId = primaryId; + this.primaryKeys = new HashSet<>(); } @Override @@ -23,12 +28,54 @@ public Map generate() { @Override protected void transformObject(Object object) { + + Class clazz = object.getClass(); + TableName annotation = clazz.getAnnotation(TableName.class); + if (annotation != null) { + this.primaryKeys.add(annotation.primaryKey()); + } + + List listFields = Arrays.stream(clazz.getDeclaredFields()) + .filter(field -> List.class.isAssignableFrom(field.getType())) + .collect(Collectors.toList()); + + for (Field field : listFields) { + try { + List list = (List) field.get(object); + for (Object o : list) { + TableName tableName = o.getClass().getAnnotation(TableName.class); + if (tableName != null && StringUtils.isNotBlank(tableName.primaryKey())) { + primaryKeys.add(tableName.primaryKey()); + break; + } + } + + } catch (IllegalAccessException e) { + e.printStackTrace(); + } + } + transformedList = Util.transformObjectCollectionToCollectionMap((List) object); setPrimaryId(); } private void setPrimaryId() { for(Map map: transformedList) { + + List collect = map.entrySet().stream() + .filter(entry -> entry.getValue() instanceof List) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + + for (String key : collect) { + if (primaryKeys.contains(key)) { + List> list =(List>) map.get(key); + for (Map stringObjectMap : list) { + stringObjectMap.put(Util.ID, stringObjectMap.get(key)); + } + } + } + map.put(Util.ID, map.get(primaryId)); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java index 87141bc9e..7d76460a2 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java @@ -1,14 +1,17 @@ package org.broadinstitute.dsm.model.elastic.migration; -import java.util.Map; +import java.lang.reflect.Field; +import java.util.*; +import java.util.stream.Collectors; +import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.OncHistoryDetail; +import org.broadinstitute.dsm.db.structure.TableName; import org.broadinstitute.dsm.statics.ESObjectConstants; public class OncHistoryDetailsMigrator extends BaseCollectionMigrator { - public OncHistoryDetailsMigrator(String index, String realm) { super(index, realm, ESObjectConstants.ONC_HISTORY_DETAIL_RECORDS, "oncHistoryDetailId"); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java index f379543ab..807833783 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java @@ -2,9 +2,13 @@ import static org.junit.Assert.*; +import java.util.Arrays; +import java.util.List; import java.util.Map; +import org.broadinstitute.dsm.db.OncHistoryDetail; import org.broadinstitute.dsm.db.Participant; +import org.broadinstitute.dsm.db.Tissue; import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; import org.junit.Test; @@ -62,4 +66,18 @@ public void transformJsonToMap() { assertEquals("TEST", result.get("ddpInstance")); assertEquals("VALUE", result.get("ddpValue")); } + + @Test + public void convertToMap() { + String fieldName = "tissue"; + List fieldValue = List.of(new Tissue("11", "22", + null, null, null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null, null, null, + null, null, null, null, null)); + + Map stringObjectMap = Util.convertToMap(fieldName, fieldValue); + + System.out.println(); + System.out.println(); + } } \ No newline at end of file From 35a869badef89f75a6e1ec3c90f16b644887c8b4 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 16 Nov 2021 16:52:40 +0400 Subject: [PATCH 119/405] DDP-7121 | test for transformObject --- .../migration/BaseCollectionMigratorTest.java | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigratorTest.java diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigratorTest.java new file mode 100644 index 000000000..4a7bedd49 --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigratorTest.java @@ -0,0 +1,47 @@ +package org.broadinstitute.dsm.model.elastic.migration; + +import static org.junit.Assert.*; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.broadinstitute.dsm.db.OncHistoryDetail; +import org.broadinstitute.dsm.db.Tissue; +import org.junit.Test; + +public class BaseCollectionMigratorTest { + + @Test + public void transformObject() { + BaseCollectionMigrator baseCollectionMigrator = new MockBaseCollectionMigrator("index", "realm", "object", "primaryId"); + baseCollectionMigrator.transformObject(something()); + System.out.println(); + } + + private List something() { + List fieldValue = new ArrayList<>(List.of(new Tissue("11", "22", + null, null, null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null, null, null, + null, null, null, null, null))); + OncHistoryDetail oncHistoryDetail = + new OncHistoryDetail(null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null, null, fieldValue, null, null, false); + return Collections.singletonList(oncHistoryDetail); + } + + static class MockBaseCollectionMigrator extends BaseCollectionMigrator { + + public MockBaseCollectionMigrator(String index, String realm, String object, String primaryId) { + super(index, realm, object, primaryId); + } + + @Override + protected Map getDataByRealm() { + return Map.of(); + } + } +} \ No newline at end of file From af64806663299a9c9e2cbe680f216e18d0f9ed31 Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 16 Nov 2021 17:24:39 +0400 Subject: [PATCH 120/405] DDP-7121 | slightly tweak BaseCollectionMigrator.java --- .../migration/BaseCollectionMigrator.java | 47 ++++++++++--------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java index a30bd3db1..8b275aee3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java @@ -28,32 +28,37 @@ public Map generate() { @Override protected void transformObject(Object object) { + Optional maybeObject = ((List) object).stream().findFirst(); + maybeObject.ifPresent((obj) -> { + Class clazz = obj.getClass(); + TableName annotation = clazz.getAnnotation(TableName.class); + if (annotation != null) { + this.primaryKeys.add(Util.underscoresToCamelCase(annotation.primaryKey())); + } - Class clazz = object.getClass(); - TableName annotation = clazz.getAnnotation(TableName.class); - if (annotation != null) { - this.primaryKeys.add(annotation.primaryKey()); - } - - List listFields = Arrays.stream(clazz.getDeclaredFields()) - .filter(field -> List.class.isAssignableFrom(field.getType())) - .collect(Collectors.toList()); + List listFields = Arrays.stream(clazz.getDeclaredFields()) + .filter(field -> List.class.isAssignableFrom(field.getType())) + .collect(Collectors.toList()); - for (Field field : listFields) { - try { - List list = (List) field.get(object); - for (Object o : list) { - TableName tableName = o.getClass().getAnnotation(TableName.class); - if (tableName != null && StringUtils.isNotBlank(tableName.primaryKey())) { - primaryKeys.add(tableName.primaryKey()); - break; + for (Field field : listFields) { + for (Object rame: (List) object) { + try { + field.setAccessible(true); + Object o = field.get(rame); + Optional first = ((List) o).stream().findFirst(); + first.ifPresent(f -> { + TableName tableName = o.getClass().getAnnotation(TableName.class); + if (tableName != null && StringUtils.isNotBlank(tableName.primaryKey())) { + primaryKeys.add(tableName.primaryKey()); + } + }); + } catch (IllegalAccessException e) { + e.printStackTrace(); } } - - } catch (IllegalAccessException e) { - e.printStackTrace(); } - } + }); + transformedList = Util.transformObjectCollectionToCollectionMap((List) object); setPrimaryId(); From 7e7faaf944599122b277403ed76486b29c9f11da Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 16 Nov 2021 17:52:46 +0400 Subject: [PATCH 121/405] DDP-7121 | add column name, fix minor issues --- .../java/org/broadinstitute/dsm/db/OncHistoryDetail.java | 1 + src/main/java/org/broadinstitute/dsm/db/Tissue.java | 1 + .../java/org/broadinstitute/dsm/model/elastic/Util.java | 4 +++- .../dsm/model/elastic/migration/BaseCollectionMigrator.java | 6 +++--- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java b/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java index 464b02453..75dc43d6b 100644 --- a/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java +++ b/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java @@ -171,6 +171,7 @@ public class OncHistoryDetail { private String participantId; + @ColumnName("tissues") private List tissues; public OncHistoryDetail(String oncHistoryDetailId, String medicalRecordId, String datePx, String typePx, diff --git a/src/main/java/org/broadinstitute/dsm/db/Tissue.java b/src/main/java/org/broadinstitute/dsm/db/Tissue.java index 9fae040ed..cd814b3e8 100644 --- a/src/main/java/org/broadinstitute/dsm/db/Tissue.java +++ b/src/main/java/org/broadinstitute/dsm/db/Tissue.java @@ -42,6 +42,7 @@ public class Tissue { alias = DBConstants.DDP_TISSUE_ALIAS, primaryKey = DBConstants.TISSUE_ID, columnPrefix = "") + @ColumnName(DBConstants.TISSUE_ID) private String tissueId; private final String oncHistoryDetailId; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index e836381df..7be1dbe2e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -130,6 +130,7 @@ static Map convertToMap(String fieldName, Object fieldValue) { for(Object obj: fieldValues) { Class inside = obj.getClass(); Field[] declaredFields = inside.getDeclaredFields(); + Map mainMap = new HashMap<>(); for (Field declaredField : declaredFields) { declaredField.setAccessible(true); ColumnName annotation = declaredField.getAnnotation(ColumnName.class); @@ -137,12 +138,13 @@ static Map convertToMap(String fieldName, Object fieldValue) { String innerFieldName = underscoresToCamelCase(annotation.value()); try { if (declaredField.get(obj) != null ) - transformedFieldValues.add(Map.of(innerFieldName, declaredField.get(obj))); + mainMap.put(innerFieldName, declaredField.get(obj)); } catch (IllegalAccessException e) { e.printStackTrace(); } } } + transformedFieldValues.add(mainMap); } fieldValue = transformedFieldValues; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java index 8b275aee3..81e35a238 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java @@ -47,9 +47,9 @@ protected void transformObject(Object object) { Object o = field.get(rame); Optional first = ((List) o).stream().findFirst(); first.ifPresent(f -> { - TableName tableName = o.getClass().getAnnotation(TableName.class); + TableName tableName = f.getClass().getAnnotation(TableName.class); if (tableName != null && StringUtils.isNotBlank(tableName.primaryKey())) { - primaryKeys.add(tableName.primaryKey()); + primaryKeys.add(Util.underscoresToCamelCase(tableName.primaryKey())); } }); } catch (IllegalAccessException e) { @@ -69,7 +69,7 @@ private void setPrimaryId() { List collect = map.entrySet().stream() .filter(entry -> entry.getValue() instanceof List) - .map(Map.Entry::getKey) + .map(entry -> ((List)entry.getValue())) .collect(Collectors.toList()); for (String key : collect) { From 90c40dbc3c250f959e96d2801d93e5e6222333b6 Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 16 Nov 2021 18:44:15 +0400 Subject: [PATCH 122/405] DDP-7121 | fix setting primary id to double-level nested objects --- .../migration/BaseCollectionMigrator.java | 24 +++++++++++++++---- .../migration/BaseCollectionMigratorTest.java | 7 ++++-- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java index 81e35a238..73976eef5 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java @@ -3,6 +3,7 @@ import java.lang.reflect.Field; import java.util.*; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.structure.TableName; @@ -67,18 +68,33 @@ protected void transformObject(Object object) { private void setPrimaryId() { for(Map map: transformedList) { +// List collect = map.entrySet().stream() +// .filter(entry -> entry.getValue() instanceof List) +// .flatMap(entry -> ((List>) entry.getValue()).stream()) +// .flatMap(entry -> entry.keySet().stream()) +// .collect(Collectors.toList()); + List collect = map.entrySet().stream() .filter(entry -> entry.getValue() instanceof List) - .map(entry -> ((List)entry.getValue())) + .map(entry -> entry.getKey()) .collect(Collectors.toList()); for (String key : collect) { - if (primaryKeys.contains(key)) { + List> o = (List>) map.get(key); + + Optional maybePrimary = o.stream() + .flatMap(mapObj -> mapObj.keySet().stream()) + .filter(k -> primaryKeys.contains(k)) + .findFirst(); + + maybePrimary.ifPresent(prKey -> { List> list =(List>) map.get(key); for (Map stringObjectMap : list) { - stringObjectMap.put(Util.ID, stringObjectMap.get(key)); + stringObjectMap.put(Util.ID, stringObjectMap.get(prKey)); } - } + }); + + } map.put(Util.ID, map.get(primaryId)); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigratorTest.java index 4a7bedd49..835d2acbb 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigratorTest.java @@ -24,8 +24,11 @@ public void transformObject() { private List something() { List fieldValue = new ArrayList<>(List.of(new Tissue("11", "22", - null, null, null, null, null, null, null, null, null, null, - null, null, null, null, null, null, null, null, null, null, + null, null, null, "awdwadawdawdawd", null, null, null, null, null, null, + null, null, "Awdawd", null, null, null, null, null, null, null, + null, null, null, null, null), new Tissue("555", "777", + null, null, null, null, null, null, null, "awdawd", null, null, + null, null, "awdawddwa", null, null, null, null, null, null, null, null, null, null, null, null))); OncHistoryDetail oncHistoryDetail = new OncHistoryDetail(null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, From 9118e9929e363715252c83f6d60a92b44867972f Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 17 Nov 2021 10:04:13 +0400 Subject: [PATCH 123/405] DDP-7121 | minor refactoring, test changes --- .../migration/BaseCollectionMigrator.java | 21 ++++++------------- .../migration/BaseCollectionMigratorTest.java | 21 +++++++++++-------- 2 files changed, 18 insertions(+), 24 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java index 73976eef5..d5d37e57e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java @@ -68,33 +68,24 @@ protected void transformObject(Object object) { private void setPrimaryId() { for(Map map: transformedList) { -// List collect = map.entrySet().stream() -// .filter(entry -> entry.getValue() instanceof List) -// .flatMap(entry -> ((List>) entry.getValue()).stream()) -// .flatMap(entry -> entry.keySet().stream()) -// .collect(Collectors.toList()); - - List collect = map.entrySet().stream() + List listValueKeys = map.entrySet().stream() .filter(entry -> entry.getValue() instanceof List) - .map(entry -> entry.getKey()) + .map(Map.Entry::getKey) .collect(Collectors.toList()); - for (String key : collect) { - List> o = (List>) map.get(key); + for (String key : listValueKeys) { + List> listValue = (List>) map.get(key); - Optional maybePrimary = o.stream() + Optional maybePrimary = listValue.stream() .flatMap(mapObj -> mapObj.keySet().stream()) .filter(k -> primaryKeys.contains(k)) .findFirst(); maybePrimary.ifPresent(prKey -> { - List> list =(List>) map.get(key); - for (Map stringObjectMap : list) { + for (Map stringObjectMap : listValue) { stringObjectMap.put(Util.ID, stringObjectMap.get(prKey)); } }); - - } map.put(Util.ID, map.get(primaryId)); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigratorTest.java index 835d2acbb..678d19f53 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigratorTest.java @@ -1,28 +1,31 @@ package org.broadinstitute.dsm.model.elastic.migration; -import static org.junit.Assert.*; - import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import org.broadinstitute.dsm.db.OncHistoryDetail; import org.broadinstitute.dsm.db.Tissue; +import org.junit.Assert; import org.junit.Test; public class BaseCollectionMigratorTest { @Test public void transformObject() { - BaseCollectionMigrator baseCollectionMigrator = new MockBaseCollectionMigrator("index", "realm", "object", "primaryId"); - baseCollectionMigrator.transformObject(something()); - System.out.println(); + BaseCollectionMigrator baseCollectionMigrator = new MockBaseCollectionMigrator("index", "realm", "object", "oncHistoryDetailId"); + baseCollectionMigrator.transformObject(mockData()); + Map objectMap = baseCollectionMigrator.transformedList.get(0); + Object primaryId = objectMap.get("id"); + Assert.assertEquals("23", primaryId); + Map tissue = (Map) ((List) objectMap.get("tissues")).get(0); + Object tissuePrimaryId = tissue.get("id"); + Assert.assertEquals("11", tissuePrimaryId); + Assert.assertEquals(tissuePrimaryId, tissue.get("tissueId")); } - private List something() { + private List mockData() { List fieldValue = new ArrayList<>(List.of(new Tissue("11", "22", null, null, null, "awdwadawdawdawd", null, null, null, null, null, null, null, null, "Awdawd", null, null, null, null, null, null, null, @@ -31,7 +34,7 @@ private List something() { null, null, "awdawddwa", null, null, null, null, null, null, null, null, null, null, null, null))); OncHistoryDetail oncHistoryDetail = - new OncHistoryDetail(null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, + new OncHistoryDetail("23", null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, fieldValue, null, null, false); return Collections.singletonList(oncHistoryDetail); } From 39d97144c072117b5d17fd8fb658fd1f94f946d2 Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 17 Nov 2021 10:31:05 +0400 Subject: [PATCH 124/405] DDP-7121 | start refactoring of BaseCollectionMigrator.java --- .../migration/BaseCollectionMigrator.java | 69 +++++++++++-------- 1 file changed, 41 insertions(+), 28 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java index d5d37e57e..3e36aee46 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java @@ -29,40 +29,53 @@ public Map generate() { @Override protected void transformObject(Object object) { - Optional maybeObject = ((List) object).stream().findFirst(); - maybeObject.ifPresent((obj) -> { - Class clazz = obj.getClass(); - TableName annotation = clazz.getAnnotation(TableName.class); - if (annotation != null) { - this.primaryKeys.add(Util.underscoresToCamelCase(annotation.primaryKey())); - } + List objects = (List) object; + Optional maybeObject = objects.stream().findFirst(); + maybeObject.ifPresent((obj) -> collectPrimaryKeys(objects, obj)); + transformedList = Util.transformObjectCollectionToCollectionMap((List) object); + setPrimaryId(); + } - List listFields = Arrays.stream(clazz.getDeclaredFields()) - .filter(field -> List.class.isAssignableFrom(field.getType())) - .collect(Collectors.toList()); + private void collectPrimaryKeys(List objects, Object obj) { + Class clazz = obj.getClass(); + TableName upperTable = clazz.getAnnotation(TableName.class); + if (hasPrimaryKey(upperTable)) { + this.primaryKeys.add(Util.underscoresToCamelCase(upperTable.primaryKey())); + } - for (Field field : listFields) { - for (Object rame: (List) object) { - try { - field.setAccessible(true); - Object o = field.get(rame); - Optional first = ((List) o).stream().findFirst(); - first.ifPresent(f -> { - TableName tableName = f.getClass().getAnnotation(TableName.class); - if (tableName != null && StringUtils.isNotBlank(tableName.primaryKey())) { - primaryKeys.add(Util.underscoresToCamelCase(tableName.primaryKey())); - } - }); - } catch (IllegalAccessException e) { - e.printStackTrace(); - } + List listFields = Arrays.stream(clazz.getDeclaredFields()) + .filter(this::isListType) + .collect(Collectors.toList()); + + for (Field field : listFields) { + // listfields = Tissue, ragaca + + field.getGenericType(); + + for (Object eachObject: objects) { + try { + field.setAccessible(true); + List fieldValue = (List) field.get(eachObject); + Optional first = fieldValue.stream().findFirst(); + first.ifPresent(instance -> { + TableName innerTable = instance.getClass().getAnnotation(TableName.class); + if (hasPrimaryKey(innerTable)) { + primaryKeys.add(Util.underscoresToCamelCase(innerTable.primaryKey())); + } + }); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); } } - }); + } + } + private boolean hasPrimaryKey(TableName table) { + return table != null && StringUtils.isNotBlank(table.primaryKey()); + } - transformedList = Util.transformObjectCollectionToCollectionMap((List) object); - setPrimaryId(); + private boolean isListType(Field field) { + return List.class.isAssignableFrom(field.getType()); } private void setPrimaryId() { From 94aa569bda9ca02216cbfd571423809a29bebb22 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 17 Nov 2021 11:06:26 +0400 Subject: [PATCH 125/405] DDP-7121 | add utility method with test to get parameterized type from field --- .../org/broadinstitute/dsm/model/elastic/Util.java | 12 ++++++++++++ .../broadinstitute/dsm/model/elastic/UtilTest.java | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 7be1dbe2e..b60c065c2 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -2,6 +2,7 @@ import java.io.IOException; import java.lang.reflect.Field; +import java.lang.reflect.Type; import java.util.*; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -190,6 +191,17 @@ private static char getFirstChar(String strValue) { return strValue.charAt(0); } + public static Class getParameterizedType(Type genericType) { + String typeAsString = genericType.toString(); + String[] types = typeAsString.split("<"); + class NullClass {} + if (types.length < 2) { + return NullClass.class; + } + String s = types[1]; + return null; + } + public static class Constants { public static final String PROFILE = "profile"; public static final String PROFILE_HRUID = PROFILE + ".hruid"; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java index 807833783..e0c811a0d 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java @@ -2,6 +2,8 @@ import static org.junit.Assert.*; +import java.lang.reflect.Field; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -80,4 +82,14 @@ public void convertToMap() { System.out.println(); System.out.println(); } + + @Test + public void getParameterizedType() throws NoSuchFieldException { + class MockClass { + List listField; + } + Field listField = MockClass.class.getDeclaredField("listField"); + Class clazz = Util.getParameterizedType(listField.getGenericType()); + assertEquals(Object.class, clazz); + } } \ No newline at end of file From a6c84658ab9f34f2e62cf60d8735bd7a6aff310d Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 17 Nov 2021 11:36:28 +0400 Subject: [PATCH 126/405] DDP-7121 | create getParameterizedType method in Util.java & start refactoring setPrimaryId() in BaseColllectionMigrator.java --- .../dsm/model/elastic/Util.java | 9 +-- .../migration/BaseCollectionMigrator.java | 69 ++++++++++--------- .../dsm/model/elastic/UtilTest.java | 8 ++- 3 files changed, 47 insertions(+), 39 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index b60c065c2..5a2af5f34 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -191,15 +191,16 @@ private static char getFirstChar(String strValue) { return strValue.charAt(0); } - public static Class getParameterizedType(Type genericType) { + public static Class getParameterizedType(Type genericType) throws ClassNotFoundException { String typeAsString = genericType.toString(); String[] types = typeAsString.split("<"); - class NullClass {} if (types.length < 2) { + class NullClass {} return NullClass.class; } - String s = types[1]; - return null; + String parameterizedType = types[1]; + parameterizedType = parameterizedType.replace(">", ""); + return Class.forName(parameterizedType); } public static class Constants { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java index 3e36aee46..7239bb691 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java @@ -3,7 +3,6 @@ import java.lang.reflect.Field; import java.util.*; import java.util.stream.Collectors; -import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.structure.TableName; @@ -31,42 +30,35 @@ public Map generate() { protected void transformObject(Object object) { List objects = (List) object; Optional maybeObject = objects.stream().findFirst(); - maybeObject.ifPresent((obj) -> collectPrimaryKeys(objects, obj)); + maybeObject.ifPresent(this::collectPrimaryKeys); transformedList = Util.transformObjectCollectionToCollectionMap((List) object); setPrimaryId(); } - private void collectPrimaryKeys(List objects, Object obj) { + private void collectPrimaryKeys(Object obj) { Class clazz = obj.getClass(); - TableName upperTable = clazz.getAnnotation(TableName.class); - if (hasPrimaryKey(upperTable)) { - this.primaryKeys.add(Util.underscoresToCamelCase(upperTable.primaryKey())); + extractAndCollectPrimaryKey(clazz); + List listFields = getListTypeFields(clazz); + for (Field field : listFields) { + try { + Class parameterizedType = Util.getParameterizedType(field.getGenericType()); + extractAndCollectPrimaryKey(parameterizedType); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } } + } - List listFields = Arrays.stream(clazz.getDeclaredFields()) - .filter(this::isListType) + private List getListTypeFields(Class clazz) { + return Arrays.stream(clazz.getDeclaredFields()) + .filter(this::isFieldListType) .collect(Collectors.toList()); + } - for (Field field : listFields) { - // listfields = Tissue, ragaca - - field.getGenericType(); - - for (Object eachObject: objects) { - try { - field.setAccessible(true); - List fieldValue = (List) field.get(eachObject); - Optional first = fieldValue.stream().findFirst(); - first.ifPresent(instance -> { - TableName innerTable = instance.getClass().getAnnotation(TableName.class); - if (hasPrimaryKey(innerTable)) { - primaryKeys.add(Util.underscoresToCamelCase(innerTable.primaryKey())); - } - }); - } catch (IllegalAccessException e) { - throw new RuntimeException(e); - } - } + private void extractAndCollectPrimaryKey(Class clazz) { + TableName upperTable = clazz.getAnnotation(TableName.class); + if (hasPrimaryKey(upperTable)) { + this.primaryKeys.add(Util.underscoresToCamelCase(upperTable.primaryKey())); } } @@ -74,19 +66,16 @@ private boolean hasPrimaryKey(TableName table) { return table != null && StringUtils.isNotBlank(table.primaryKey()); } - private boolean isListType(Field field) { + private boolean isFieldListType(Field field) { return List.class.isAssignableFrom(field.getType()); } private void setPrimaryId() { for(Map map: transformedList) { - - List listValueKeys = map.entrySet().stream() - .filter(entry -> entry.getValue() instanceof List) - .map(Map.Entry::getKey) - .collect(Collectors.toList()); + List listValueKeys = getListValueKeys(map); for (String key : listValueKeys) { + // Tissue, Details -> 1 to many List> listValue = (List>) map.get(key); Optional maybePrimary = listValue.stream() @@ -104,4 +93,16 @@ private void setPrimaryId() { map.put(Util.ID, map.get(primaryId)); } } + + private List getListValueKeys(Map map) { + return map.entrySet().stream() + .filter(this::isMapValueListType) + .filter(entry -> primaryKeys.contains(entry.getKey())) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + } + + private boolean isMapValueListType(Map.Entry entry) { + return entry.getValue() instanceof List; + } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java index e0c811a0d..a3c5a8ad1 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java @@ -12,6 +12,7 @@ import org.broadinstitute.dsm.db.Participant; import org.broadinstitute.dsm.db.Tissue; import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; +import org.junit.Assert; import org.junit.Test; public class UtilTest { @@ -89,7 +90,12 @@ class MockClass { List listField; } Field listField = MockClass.class.getDeclaredField("listField"); - Class clazz = Util.getParameterizedType(listField.getGenericType()); + Class clazz = null; + try { + clazz = Util.getParameterizedType(listField.getGenericType()); + } catch (ClassNotFoundException e) { + Assert.fail(); + } assertEquals(Object.class, clazz); } } \ No newline at end of file From ca291aa315e44e4fc4c33553f5b40ba56e2ec72d Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 17 Nov 2021 16:33:22 +0400 Subject: [PATCH 127/405] DDP-7121 | start reducing duplication in BaseCollectionMigrator.java --- .../migration/BaseCollectionMigrator.java | 36 +++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java index 7239bb691..e7450f648 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java @@ -73,31 +73,47 @@ private boolean isFieldListType(Field field) { private void setPrimaryId() { for(Map map: transformedList) { List listValueKeys = getListValueKeys(map); - for (String key : listValueKeys) { - // Tissue, Details -> 1 to many List> listValue = (List>) map.get(key); - - Optional maybePrimary = listValue.stream() - .flatMap(mapObj -> mapObj.keySet().stream()) - .filter(k -> primaryKeys.contains(k)) - .findFirst(); - + Optional maybePrimary = getPrimaryKey(listValue); maybePrimary.ifPresent(prKey -> { for (Map stringObjectMap : listValue) { stringObjectMap.put(Util.ID, stringObjectMap.get(prKey)); } }); } + Optional maybeOuterKey = map.keySet().stream() + .filter(outerKey -> primaryKeys.contains(outerKey)) + .findFirst(); + maybeOuterKey.ifPresent(outerKey -> { + map.put(Util.ID, map.get(outerKey)); + }); + } + } + + private Optional getPrimaryKey(List> listValue) { - map.put(Util.ID, map.get(primaryId)); + for(Map eachValue: listValue) { + } + + + return listValue.stream() + .flatMap(mapObj -> mapObj.keySet().stream()) + .filter(k -> primaryKeys.contains(k)) + .findFirst(); } + private Optional method(Map map) { + return map.keySet().stream() + .filter(outerKey -> primaryKeys.contains(outerKey)) + .findFirst(); + } + + private List getListValueKeys(Map map) { return map.entrySet().stream() .filter(this::isMapValueListType) - .filter(entry -> primaryKeys.contains(entry.getKey())) .map(Map.Entry::getKey) .collect(Collectors.toList()); } From d8e6f1bc509c31746ee69afd85f185b5ac82b4be Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 17 Nov 2021 16:55:48 +0400 Subject: [PATCH 128/405] DDP-7121 | remove unnecessary passing of primary keys --- .../migration/BaseCollectionMigrator.java | 33 ++++++++----------- .../migration/KitRequestShippingMigrator.java | 2 +- .../migration/MedicalRecordMigrator.java | 2 +- .../migration/OncHistoryDetailsMigrator.java | 2 +- .../migration/ParticipantDataMigrator.java | 2 +- 5 files changed, 17 insertions(+), 24 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java index e7450f648..2071b33a1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java @@ -11,13 +11,11 @@ public abstract class BaseCollectionMigrator extends BaseMigrator { - protected String primaryId; protected List> transformedList; protected Set primaryKeys; - public BaseCollectionMigrator(String index, String realm, String object, String primaryId) { + public BaseCollectionMigrator(String index, String realm, String object) { super(index, realm, object); - this.primaryId = primaryId; this.primaryKeys = new HashSet<>(); } @@ -75,36 +73,31 @@ private void setPrimaryId() { List listValueKeys = getListValueKeys(map); for (String key : listValueKeys) { List> listValue = (List>) map.get(key); - Optional maybePrimary = getPrimaryKey(listValue); + Optional maybePrimary = getPrimaryKeyFromList(listValue); maybePrimary.ifPresent(prKey -> { for (Map stringObjectMap : listValue) { - stringObjectMap.put(Util.ID, stringObjectMap.get(prKey)); + putPrimaryId(stringObjectMap, prKey); } }); } - Optional maybeOuterKey = map.keySet().stream() - .filter(outerKey -> primaryKeys.contains(outerKey)) - .findFirst(); - maybeOuterKey.ifPresent(outerKey -> { - map.put(Util.ID, map.get(outerKey)); - }); + getPrimaryKey(map) + .ifPresent(outerKey -> putPrimaryId(map, outerKey)); } } - private Optional getPrimaryKey(List> listValue) { + private void putPrimaryId(Map map, String outerKey) { + map.put(Util.ID, map.get(outerKey)); + } + private Optional getPrimaryKeyFromList(List> listValue) { for(Map eachValue: listValue) { - + Optional maybePrimaryKey = getPrimaryKey(eachValue); + if (maybePrimaryKey.isPresent()) return maybePrimaryKey; } - - - return listValue.stream() - .flatMap(mapObj -> mapObj.keySet().stream()) - .filter(k -> primaryKeys.contains(k)) - .findFirst(); + return Optional.empty(); } - private Optional method(Map map) { + private Optional getPrimaryKey(Map map) { return map.keySet().stream() .filter(outerKey -> primaryKeys.contains(outerKey)) .findFirst(); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/KitRequestShippingMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/KitRequestShippingMigrator.java index bba3f8fa5..ded3df947 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/KitRequestShippingMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/KitRequestShippingMigrator.java @@ -9,7 +9,7 @@ public class KitRequestShippingMigrator extends BaseCollectionMigrator { public KitRequestShippingMigrator(String index, String realm) { - super(index, realm, ESObjectConstants.KIT_REQUEST_SHIPPING, "dsmKitRequestId"); + super(index, realm, ESObjectConstants.KIT_REQUEST_SHIPPING); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigrator.java index 2478d3e68..c34197469 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigrator.java @@ -8,7 +8,7 @@ public class MedicalRecordMigrator extends BaseCollectionMigrator { public MedicalRecordMigrator(String index, String realm) { - super(index, realm, ESObjectConstants.MEDICAL_RECORDS, "medicalRecordId"); + super(index, realm, ESObjectConstants.MEDICAL_RECORDS); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java index 7d76460a2..b04aba278 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java @@ -13,7 +13,7 @@ public class OncHistoryDetailsMigrator extends BaseCollectionMigrator { public OncHistoryDetailsMigrator(String index, String realm) { - super(index, realm, ESObjectConstants.ONC_HISTORY_DETAIL_RECORDS, "oncHistoryDetailId"); + super(index, realm, ESObjectConstants.ONC_HISTORY_DETAIL_RECORDS); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java index 4d310395d..3f6ebdd50 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java @@ -23,7 +23,7 @@ public class ParticipantDataMigrator extends BaseCollectionMigrator { private ParticipantDataDao participantDataDao; public ParticipantDataMigrator(String index, String realm) { - super(index, realm, ESObjectConstants.PARTICIPANT_DATA, "participantDataId"); + super(index, realm, ESObjectConstants.PARTICIPANT_DATA); participantDataDao = new ParticipantDataDao(); } From 4bb23134875b4ff089ecb1972887a1940d79af2a Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 17 Nov 2021 17:15:46 +0400 Subject: [PATCH 129/405] DDP-7121 | rename instance fields to match column names as camelCase, reorganize methods to provide readability --- .../org/broadinstitute/dsm/db/Tissue.java | 26 +++++++------- .../broadinstitute/dsm/model/TissueList.java | 2 +- .../migration/BaseCollectionMigrator.java | 35 +++++++++---------- .../dsm/statics/DBConstants.java | 1 - 4 files changed, 31 insertions(+), 33 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/Tissue.java b/src/main/java/org/broadinstitute/dsm/db/Tissue.java index cd814b3e8..53db6713d 100644 --- a/src/main/java/org/broadinstitute/dsm/db/Tissue.java +++ b/src/main/java/org/broadinstitute/dsm/db/Tissue.java @@ -48,7 +48,7 @@ public class Tissue { private final String oncHistoryDetailId; @ColumnName (DBConstants.NOTES) - private final String tNotes; + private final String notes; @ColumnName (DBConstants.COUNT_RECEIVED) private final Integer countReceived; @@ -93,18 +93,18 @@ public class Tissue { private String changedBy; - @ColumnName (DBConstants.TDELETED) - private boolean tDeleted; + @ColumnName (DBConstants.DELETED) + private boolean deleted; @ColumnName (DBConstants.FIRST_SM_ID) private String firstSmId; @ColumnName (DBConstants.ADDITIONAL_TISSUE_VALUES) - private String additionalValues; + private String additionalTissueValueJson; @ColumnName (DBConstants.TISSUE_RETURN_DATE) @DbDateConversion(SqlDateConverter.STRING_DAY) - private String tissueReturnDate; + private String returnDate; // @ColumnName (DBConstants.RETURN_FEDEX_ID) private String returnFedexId; @@ -116,7 +116,7 @@ public class Tissue { private String tumorPercentage; @ColumnName (DBConstants.TISSUE_SEQUENCE) - private String sequenceResults; + private String tissueSequence; @ColumnName (DBConstants.SCROLLS_COUNT) private Integer scrollsCount; @@ -132,15 +132,15 @@ public class Tissue { - public Tissue(String tissueId, String oncHistoryDetailId, String tNotes, Integer countReceived, String tissueType, + public Tissue(String tissueId, String oncHistoryDetailId, String notes, Integer countReceived, String tissueType, String tissueSite, String tumorType, String hE, String pathologyReport, String collaboratorSampleId, String blockSent, String scrollsReceived, String skId, String smId, String sentGp, String firstSmId, - String additionalValues, String expectedReturn, String tissueReturnDate, - String returnFedexId, String shlWorkNumber, String tumorPercentage, String sequenceResults, Integer scrollsCount, + String additionalTissueValueJson, String expectedReturn, String returnDate, + String returnFedexId, String shlWorkNumber, String tumorPercentage, String tissueSequence, Integer scrollsCount, Integer ussCount, Integer blocksCount, Integer hECount) { this.tissueId = tissueId; this.oncHistoryDetailId = oncHistoryDetailId; - this.tNotes = tNotes; + this.notes = notes; this.countReceived = countReceived; this.tissueType = tissueType; this.tissueSite = tissueSite; @@ -154,13 +154,13 @@ public Tissue(String tissueId, String oncHistoryDetailId, String tNotes, Integer this.smId = smId; this.sentGp = sentGp; this.firstSmId = firstSmId; - this.additionalValues = additionalValues; + this.additionalTissueValueJson = additionalTissueValueJson; this.expectedReturn = expectedReturn; - this.tissueReturnDate = tissueReturnDate; + this.returnDate = returnDate; this.returnFedexId = returnFedexId; this.shlWorkNumber = shlWorkNumber; this.tumorPercentage = tumorPercentage; - this.sequenceResults = sequenceResults; + this.tissueSequence = tissueSequence; this.scrollsCount = scrollsCount; this.hECount = hECount; this.blocksCount = blocksCount; diff --git a/src/main/java/org/broadinstitute/dsm/model/TissueList.java b/src/main/java/org/broadinstitute/dsm/model/TissueList.java index 2e9c676ef..a3e5220d4 100644 --- a/src/main/java/org/broadinstitute/dsm/model/TissueList.java +++ b/src/main/java/org/broadinstitute/dsm/model/TissueList.java @@ -64,7 +64,7 @@ public static List getAllTissueListsForRealm(String realm, String qu Tissue tissue = Tissue.getTissue(rs); TissueList tissueList = new TissueList(oncHistory, null, ptId); - if (!tissue.isTDeleted() && StringUtils.isNotBlank(tissue.getTissueId())) { + if (!tissue.isDeleted() && StringUtils.isNotBlank(tissue.getTissueId())) { tissueList.setTissue(tissue); } results.add(tissueList); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java index 2071b33a1..4aca1f26b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java @@ -47,12 +47,6 @@ private void collectPrimaryKeys(Object obj) { } } - private List getListTypeFields(Class clazz) { - return Arrays.stream(clazz.getDeclaredFields()) - .filter(this::isFieldListType) - .collect(Collectors.toList()); - } - private void extractAndCollectPrimaryKey(Class clazz) { TableName upperTable = clazz.getAnnotation(TableName.class); if (hasPrimaryKey(upperTable)) { @@ -64,6 +58,12 @@ private boolean hasPrimaryKey(TableName table) { return table != null && StringUtils.isNotBlank(table.primaryKey()); } + private List getListTypeFields(Class clazz) { + return Arrays.stream(clazz.getDeclaredFields()) + .filter(this::isFieldListType) + .collect(Collectors.toList()); + } + private boolean isFieldListType(Field field) { return List.class.isAssignableFrom(field.getType()); } @@ -85,8 +85,15 @@ private void setPrimaryId() { } } - private void putPrimaryId(Map map, String outerKey) { - map.put(Util.ID, map.get(outerKey)); + private List getListValueKeys(Map map) { + return map.entrySet().stream() + .filter(this::isMapValueListType) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + } + + private boolean isMapValueListType(Map.Entry entry) { + return entry.getValue() instanceof List; } private Optional getPrimaryKeyFromList(List> listValue) { @@ -103,15 +110,7 @@ private Optional getPrimaryKey(Map map) { .findFirst(); } - - private List getListValueKeys(Map map) { - return map.entrySet().stream() - .filter(this::isMapValueListType) - .map(Map.Entry::getKey) - .collect(Collectors.toList()); - } - - private boolean isMapValueListType(Map.Entry entry) { - return entry.getValue() instanceof List; + private void putPrimaryId(Map map, String outerKey) { + map.put(Util.ID, map.get(outerKey)); } } diff --git a/src/main/java/org/broadinstitute/dsm/statics/DBConstants.java b/src/main/java/org/broadinstitute/dsm/statics/DBConstants.java index 4b81f9818..291397e1b 100644 --- a/src/main/java/org/broadinstitute/dsm/statics/DBConstants.java +++ b/src/main/java/org/broadinstitute/dsm/statics/DBConstants.java @@ -212,7 +212,6 @@ public class DBConstants { public static final String TISSUE_RECEIVED = "tissue_received"; public static final String TISSUE_PROBLEM_OPTION = "tissue_problem_option" ; public static final String GENDER = "gender"; - public static final String TDELETED = "tDeleted"; public static final String DESTRUCTION_POLICY = "destruction_policy"; public static final String UNABLE_OBTAIN_TISSUE = "unable_obtain_tissue"; From 1e5d61180d164ba1095416fe9f0ec1048c11809e Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 17 Nov 2021 18:42:57 +0400 Subject: [PATCH 130/405] DDP-7121 | add checks for nulls, change test based on new constructor, remove primaryKeyId-s --- .../java/org/broadinstitute/dsm/db/OncHistoryDetail.java | 6 ++++-- src/main/java/org/broadinstitute/dsm/db/Tissue.java | 4 ++++ .../model/elastic/migration/OncHistoryDetailsMigrator.java | 2 +- .../org/broadinstitute/dsm/statics/ESObjectConstants.java | 1 + .../model/elastic/migration/BaseCollectionMigratorTest.java | 6 +++--- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java b/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java index 75dc43d6b..96833e34a 100644 --- a/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java +++ b/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java @@ -328,11 +328,13 @@ public static Map> getOncHistoryDetails(@NonNull OncHistoryDetail oncHistoryDetail = null; if (oncHistoryMap.containsKey(oncHistoryDetailId)) { oncHistoryDetail = oncHistoryMap.get(oncHistoryDetailId); - oncHistoryDetail.addTissue(tissue); + if (tissue != null) + oncHistoryDetail.addTissue(tissue); } else { oncHistoryDetail = getOncHistoryDetail(rs); - oncHistoryDetail.addTissue(tissue); + if (tissue != null) + oncHistoryDetail.addTissue(tissue); oncHistoryDataList.add(oncHistoryDetail); } oncHistoryMap.put(oncHistoryDetailId, oncHistoryDetail); diff --git a/src/main/java/org/broadinstitute/dsm/db/Tissue.java b/src/main/java/org/broadinstitute/dsm/db/Tissue.java index 53db6713d..1472ce0dc 100644 --- a/src/main/java/org/broadinstitute/dsm/db/Tissue.java +++ b/src/main/java/org/broadinstitute/dsm/db/Tissue.java @@ -2,6 +2,7 @@ import lombok.Data; import lombok.NonNull; +import org.apache.commons.lang3.StringUtils; import org.broadinstitute.ddp.db.SimpleResult; import org.broadinstitute.dsm.db.structure.ColumnName; import org.broadinstitute.dsm.db.structure.DbDateConversion; @@ -168,6 +169,9 @@ public Tissue(String tissueId, String oncHistoryDetailId, String notes, Integer } public static Tissue getTissue(@NonNull ResultSet rs) throws SQLException { + String tissueId = rs.getString(DBConstants.TISSUE_ID); + if (StringUtils.isBlank(tissueId)) + return null; Tissue tissue = new Tissue( rs.getString(DBConstants.TISSUE_ID), rs.getString(DBConstants.ONC_HISTORY_DETAIL_ID), diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java index b04aba278..6e90f2100 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java @@ -13,7 +13,7 @@ public class OncHistoryDetailsMigrator extends BaseCollectionMigrator { public OncHistoryDetailsMigrator(String index, String realm) { - super(index, realm, ESObjectConstants.ONC_HISTORY_DETAIL_RECORDS); + super(index, realm, ESObjectConstants.ONC_HISTORY_DETAILS); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java b/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java index 6487a2b9c..e677cafb5 100644 --- a/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java +++ b/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java @@ -66,6 +66,7 @@ public class ESObjectConstants { public static final String HAS_CONSENTED_TO_TISSUE_SAMPLE = "hasConsentedToTissueSample"; public static final String DIAGNOSIS_YEAR = "diagnosisYear"; public static final String ONC_HISTORY_DETAIL_RECORDS = "oncHistoryDetailRecords"; + public static final String ONC_HISTORY_DETAILS = "oncHistoryDetails"; public static final String ONC_HISTORY = "oncHistory"; public static final String PARTICIPANT_DATA = "participantData"; public static final String PARTICIPANT_RECORD = "participantRecord"; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigratorTest.java index 678d19f53..3b44b6b7b 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigratorTest.java @@ -14,7 +14,7 @@ public class BaseCollectionMigratorTest { @Test public void transformObject() { - BaseCollectionMigrator baseCollectionMigrator = new MockBaseCollectionMigrator("index", "realm", "object", "oncHistoryDetailId"); + BaseCollectionMigrator baseCollectionMigrator = new MockBaseCollectionMigrator("index", "realm", "object"); baseCollectionMigrator.transformObject(mockData()); Map objectMap = baseCollectionMigrator.transformedList.get(0); Object primaryId = objectMap.get("id"); @@ -41,8 +41,8 @@ private List mockData() { static class MockBaseCollectionMigrator extends BaseCollectionMigrator { - public MockBaseCollectionMigrator(String index, String realm, String object, String primaryId) { - super(index, realm, object, primaryId); + public MockBaseCollectionMigrator(String index, String realm, String object) { + super(index, realm, object); } @Override From 0a1e082da326c913d676da35d8e56fe35b53f3e0 Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 18 Nov 2021 16:59:43 +0400 Subject: [PATCH 131/405] DDP-7121 | refactor FollowUp from array to List --- .../broadinstitute/dsm/db/MedicalRecord.java | 6 +++--- .../broadinstitute/dsm/model/FollowUp.java | 9 +++++++++ .../dsm/model/elastic/Util.java | 3 ++- .../elastic/migration/BulkExportFacade.java | 1 + .../dsm/route/DashboardRoute.java | 2 +- .../dsm/model/elastic/UtilTest.java | 8 +++++++- .../migration/MedicalRecordMigratorTest.java | 20 +++++++++++++++++++ 7 files changed, 43 insertions(+), 6 deletions(-) create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigratorTest.java diff --git a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java index b418f3339..022a20c65 100644 --- a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java +++ b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java @@ -306,7 +306,7 @@ public class MedicalRecord { primaryKey = DBConstants.MEDICAL_RECORD_ID, columnPrefix = "") @ColumnName (DBConstants.FOLLOW_UP_REQUESTS) - private FollowUp[] followUps; + private List followUps; @TableName ( name = DBConstants.DDP_MEDICAL_RECORD, @@ -341,7 +341,7 @@ public MedicalRecord(String medicalRecordId, String institutionId, String ddpIns String mrReceived, String mrDocument, String mrDocumentFileNames, boolean mrProblem, String mrProblemText, boolean unableObtain, boolean duplicate, boolean international, boolean crRequired, String pathologyPresent, String mrNotes, boolean reviewMedicalRecord, - FollowUp[] followUps, boolean followUpRequired, String followUpRequiredText, String additionalValues, + List followUps, boolean followUpRequired, String followUpRequiredText, String additionalValues, String mrUnableToObtainText, String ddpParticipantId) { this.medicalRecordId = medicalRecordId; this.institutionId = institutionId; @@ -411,7 +411,7 @@ public static MedicalRecord getMedicalRecord(@NonNull ResultSet rs) throws SQLEx rs.getString(DBConstants.PATHOLOGY_PRESENT), rs.getString(DBConstants.NOTES), rs.getBoolean(DBConstants.REVIEW_MEDICAL_RECORD), - new Gson().fromJson(rs.getString(DBConstants.FOLLOW_UP_REQUESTS), FollowUp[].class), + new Gson().fromJson(rs.getString(DBConstants.FOLLOW_UP_REQUESTS), List.class), rs.getBoolean(DBConstants.FOLLOWUP_REQUIRED), rs.getString(DBConstants.FOLLOWUP_REQUIRED_TEXT), rs.getString(DBConstants.ADDITIONAL_VALUES), diff --git a/src/main/java/org/broadinstitute/dsm/model/FollowUp.java b/src/main/java/org/broadinstitute/dsm/model/FollowUp.java index 0c2f154c7..dcc09833e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/FollowUp.java +++ b/src/main/java/org/broadinstitute/dsm/model/FollowUp.java @@ -1,12 +1,21 @@ package org.broadinstitute.dsm.model; import lombok.Data; +import org.broadinstitute.dsm.db.structure.ColumnName; @Data public class FollowUp { + + @ColumnName("fReceived") private String fReceived; + + @ColumnName("fRequest1") private String fRequest1; + + @ColumnName("fRequest2") private String fRequest2; + + @ColumnName("fRequest3") private String fRequest3; public FollowUp(String fRequest1, String fRequest2, String fRequest3, String fReceived) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 5a2af5f34..c9e34a064 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -193,13 +193,14 @@ private static char getFirstChar(String strValue) { public static Class getParameterizedType(Type genericType) throws ClassNotFoundException { String typeAsString = genericType.toString(); - String[] types = typeAsString.split("<"); + String[] types = typeAsString.contains("<") ? typeAsString.split("<") : typeAsString.split("\\[L"); if (types.length < 2) { class NullClass {} return NullClass.class; } String parameterizedType = types[1]; parameterizedType = parameterizedType.replace(">", ""); + parameterizedType = parameterizedType.replace(";", ""); return Class.forName(parameterizedType); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BulkExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BulkExportFacade.java index 0c0e0cb17..39d1b446e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BulkExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BulkExportFacade.java @@ -3,6 +3,7 @@ import static org.broadinstitute.dsm.model.elastic.Util.DOC; import java.io.IOException; +import java.util.List; import java.util.Map; import org.broadinstitute.dsm.util.ElasticSearchUtil; diff --git a/src/main/java/org/broadinstitute/dsm/route/DashboardRoute.java b/src/main/java/org/broadinstitute/dsm/route/DashboardRoute.java index cfda12875..4480ca44f 100644 --- a/src/main/java/org/broadinstitute/dsm/route/DashboardRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/DashboardRoute.java @@ -362,7 +362,7 @@ private void countMedicalRecordData(@NonNull List medicalRecordLi if (medicalRecord.isFollowUpRequired()) { incrementCounter(dashboardValuesDetailed, "followupRequiredMedicalRecord"); foundAtPT.add("followupRequiredMedicalRecord"); - if (medicalRecord.getFollowUps() == null || medicalRecord.getFollowUps().length == 0) { + if (medicalRecord.getFollowUps() == null || medicalRecord.getFollowUps().size() == 0) { incrementCounter(dashboardValuesDetailed, "followupNotRequested"); foundAtPT.add("followupNotRequested"); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java index a3c5a8ad1..995a7f453 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java @@ -12,6 +12,7 @@ import org.broadinstitute.dsm.db.Participant; import org.broadinstitute.dsm.db.Tissue; import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; +import org.broadinstitute.dsm.model.FollowUp; import org.junit.Assert; import org.junit.Test; @@ -88,14 +89,19 @@ public void convertToMap() { public void getParameterizedType() throws NoSuchFieldException { class MockClass { List listField; + FollowUp[] followUps; } Field listField = MockClass.class.getDeclaredField("listField"); + Field followUps = MockClass.class.getDeclaredField("followUps"); + Class clazz = null; try { clazz = Util.getParameterizedType(listField.getGenericType()); + assertEquals(Object.class, clazz); + clazz = Util.getParameterizedType(followUps.getGenericType()); + assertEquals(FollowUp.class, clazz); } catch (ClassNotFoundException e) { Assert.fail(); } - assertEquals(Object.class, clazz); } } \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigratorTest.java new file mode 100644 index 000000000..2971c7600 --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigratorTest.java @@ -0,0 +1,20 @@ +package org.broadinstitute.dsm.model.elastic.migration; + +import org.broadinstitute.dsm.TestHelper; +import org.junit.BeforeClass; +import org.junit.Test; + +public class MedicalRecordMigratorTest { + + @BeforeClass + public static void setUp() { + TestHelper.setupDB(); + } + + @Test + public void export() { + MedicalRecordMigrator medicalRecordMigrator = new MedicalRecordMigrator("participants_structured.cmi.angio", "angio"); + medicalRecordMigrator.export(); + } + +} \ No newline at end of file From c08212e6c85d60348f4830017339ff4ece8077f6 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 18 Nov 2021 17:30:03 +0400 Subject: [PATCH 132/405] DDP-7121 | add type token to specify class --- src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java | 3 ++- .../java/org/broadinstitute/dsm/model/elastic/Util.java | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java index 022a20c65..3b890a325 100644 --- a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java +++ b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java @@ -1,5 +1,6 @@ package org.broadinstitute.dsm.db; +import com.google.common.reflect.TypeToken; import com.google.gson.Gson; import lombok.Data; import lombok.NonNull; @@ -411,7 +412,7 @@ public static MedicalRecord getMedicalRecord(@NonNull ResultSet rs) throws SQLEx rs.getString(DBConstants.PATHOLOGY_PRESENT), rs.getString(DBConstants.NOTES), rs.getBoolean(DBConstants.REVIEW_MEDICAL_RECORD), - new Gson().fromJson(rs.getString(DBConstants.FOLLOW_UP_REQUESTS), List.class), + new Gson().fromJson(rs.getString(DBConstants.FOLLOW_UP_REQUESTS), new TypeToken>(){}.getType()), rs.getBoolean(DBConstants.FOLLOWUP_REQUIRED), rs.getString(DBConstants.FOLLOWUP_REQUIRED_TEXT), rs.getString(DBConstants.ADDITIONAL_VALUES), diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index c9e34a064..57dbc2362 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -110,9 +110,9 @@ public static Map transformObjectToMap(Object obj) { static Map convertToMap(String fieldName, Object fieldValue) { Map finalResult; switch (fieldName) { - case "follow_ups": - finalResult = Map.of(underscoresToCamelCase(fieldName), new Gson().toJson(fieldValue)); - break; +// case "follow_ups": +// finalResult = Map.of(underscoresToCamelCase(fieldName), new Gson().toJson(fieldValue)); +// break; case "data": Map objectMap = dynamicFieldsSpecialCase(fieldValue); Map transformedMap = new HashMap<>(); From 471d5f938446323918401716bc998d6831c11e3b Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 18 Nov 2021 18:15:31 +0400 Subject: [PATCH 133/405] DDP-7121 | restrict mapping update based on type of data --- .../dsm/model/elastic/export/ExportFacade.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 1641d1e1f..4d9334ce1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -43,11 +43,14 @@ public void export() { } private void upsertMapping() { - generator = new MappingGenerator(new TypeParser(), exportFacadePayload.getGeneratorPayload()); - Map mappingToUpsert = generator.generate(); - UpsertMappingRequestPayload upsertMappingRequestPayload = new UpsertMappingRequestPayload(exportFacadePayload.getIndex()); - exportable = new ElasticMappingExportAdapter(upsertMappingRequestPayload, mappingToUpsert); - exportable.export(); + boolean dynamic = exportFacadePayload.getGeneratorPayload().getNameValue().getName().contains("data"); + if (dynamic) { + generator = new MappingGenerator(new TypeParser(), exportFacadePayload.getGeneratorPayload()); + Map mappingToUpsert = generator.generate(); + UpsertMappingRequestPayload upsertMappingRequestPayload = new UpsertMappingRequestPayload(exportFacadePayload.getIndex()); + exportable = new ElasticMappingExportAdapter(upsertMappingRequestPayload, mappingToUpsert); + exportable.export(); + } } private ESDsm fetchData() { From ebe107ad09d22e515b2db06187204e2ff5e4dec0 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 19 Nov 2021 10:35:25 +0400 Subject: [PATCH 134/405] DDP-7121 | add tissue migrator, start of changing mapping generator --- .../org/broadinstitute/dsm/db/Tissue.java | 1 + .../dsm/db/dao/ddp/tissue/TissueDao.java | 81 +++++++++++++++++++ .../dsm/model/elastic/Util.java | 3 - .../model/elastic/export/ExportFacade.java | 13 ++- .../elastic/export/parse/BaseParser.java | 20 +++++ .../elastic/export/parse/TypeParser.java | 15 ++++ .../elastic/export/parse/ValueParser.java | 5 ++ .../elastic/migration/BulkExportFacade.java | 1 + .../migration/OncHistoryDetailsMigrator.java | 6 +- .../elastic/migration/TissueMigrator.java | 17 ++++ .../elastic/export/parse/BaseParserTest.java | 27 +++++++ .../elastic/migration/TissueMigratorTest.java | 23 ++++++ 12 files changed, 196 insertions(+), 16 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/db/dao/ddp/tissue/TissueDao.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigrator.java create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserTest.java create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigratorTest.java diff --git a/src/main/java/org/broadinstitute/dsm/db/Tissue.java b/src/main/java/org/broadinstitute/dsm/db/Tissue.java index 1472ce0dc..f403267d5 100644 --- a/src/main/java/org/broadinstitute/dsm/db/Tissue.java +++ b/src/main/java/org/broadinstitute/dsm/db/Tissue.java @@ -46,6 +46,7 @@ public class Tissue { @ColumnName(DBConstants.TISSUE_ID) private String tissueId; + @ColumnName(DBConstants.ONC_HISTORY_DETAIL_ID) private final String oncHistoryDetailId; @ColumnName (DBConstants.NOTES) diff --git a/src/main/java/org/broadinstitute/dsm/db/dao/ddp/tissue/TissueDao.java b/src/main/java/org/broadinstitute/dsm/db/dao/ddp/tissue/TissueDao.java new file mode 100644 index 000000000..287364ab5 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/db/dao/ddp/tissue/TissueDao.java @@ -0,0 +1,81 @@ +package org.broadinstitute.dsm.db.dao.ddp.tissue; + +import static org.broadinstitute.ddp.db.TransactionWrapper.inTransaction; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +import org.broadinstitute.ddp.db.SimpleResult; +import org.broadinstitute.dsm.db.Tissue; +import org.broadinstitute.dsm.db.dao.Dao; +import org.broadinstitute.dsm.statics.DBConstants; + +public class TissueDao implements Dao { + + + private static final String SQL_SELECT_TISSUES_BY_STUDY = "SELECT p.ddp_participant_id, " + + "tissue_id, t.onc_history_detail_id, t.notes, count_received, tissue_type, tissue_site, tumor_type, h_e, pathology_report, " + + "collaborator_sample_id, " + + "block_sent, scrolls_received, sk_id, sm_id, sent_gp, first_sm_id, additional_tissue_value_json, expected_return, return_date, " + + "return_fedex_id, shl_work_number, tumor_percentage, tissue_sequence, scrolls_count, uss_count, h_e_count, blocks_count " + + "FROM ddp_onc_history_detail oD " + + "LEFT JOIN ddp_medical_record m on (oD.medical_record_id = m.medical_record_id AND NOT oD.deleted <=> 1 AND NOT m.deleted <=> 1) " + + "LEFT JOIN ddp_institution inst on (inst.institution_id = m.institution_id) " + + "LEFT JOIN ddp_participant p on (p.participant_id = inst.participant_id) " + + "LEFT JOIN ddp_instance realm on (p.ddp_instance_id = realm.ddp_instance_id) " + + "LEFT JOIN ddp_tissue t on (oD.onc_history_detail_id = t.onc_history_detail_id AND NOT t.deleted <=> 1) " + + "WHERE realm.instance_name = ? "; + + @Override + public int create(Tissue tissue) { + return 0; + } + + @Override + public int delete(int id) { + return 0; + } + + @Override + public Optional get(long id) { + return Optional.empty(); + } + + public Map> getTissuesByStudy(String study) { + Map> tissues = new HashMap<>(); + SimpleResult results = inTransaction((conn) -> { + SimpleResult execResult = new SimpleResult(); + try (PreparedStatement stmt = conn.prepareStatement(SQL_SELECT_TISSUES_BY_STUDY)) { + stmt.setString(1, study); + try(ResultSet rs = stmt.executeQuery()) { + while (rs.next()) { + String ddpParticipantId = rs.getString(DBConstants.DDP_PARTICIPANT_ID); + Tissue tissue = Tissue.getTissue(rs); + if (Objects.isNull(tissue)) continue; + tissues.merge(ddpParticipantId, new ArrayList<>(List.of(tissue)), (prev, curr) -> { + prev.addAll(curr); + return prev; + }); + } + } + } + catch (SQLException ex) { + execResult.resultException = ex; + } + return execResult; + }); + if (results.resultException != null) { + throw new RuntimeException("Error getting tissues for " + + study, results.resultException); + } + return tissues; + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 57dbc2362..b47277809 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -110,9 +110,6 @@ public static Map transformObjectToMap(Object obj) { static Map convertToMap(String fieldName, Object fieldValue) { Map finalResult; switch (fieldName) { -// case "follow_ups": -// finalResult = Map.of(underscoresToCamelCase(fieldName), new Gson().toJson(fieldValue)); -// break; case "data": Map objectMap = dynamicFieldsSpecialCase(fieldValue); Map transformedMap = new HashMap<>(); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 4d9334ce1..1641d1e1f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -43,14 +43,11 @@ public void export() { } private void upsertMapping() { - boolean dynamic = exportFacadePayload.getGeneratorPayload().getNameValue().getName().contains("data"); - if (dynamic) { - generator = new MappingGenerator(new TypeParser(), exportFacadePayload.getGeneratorPayload()); - Map mappingToUpsert = generator.generate(); - UpsertMappingRequestPayload upsertMappingRequestPayload = new UpsertMappingRequestPayload(exportFacadePayload.getIndex()); - exportable = new ElasticMappingExportAdapter(upsertMappingRequestPayload, mappingToUpsert); - exportable.export(); - } + generator = new MappingGenerator(new TypeParser(), exportFacadePayload.getGeneratorPayload()); + Map mappingToUpsert = generator.generate(); + UpsertMappingRequestPayload upsertMappingRequestPayload = new UpsertMappingRequestPayload(exportFacadePayload.getIndex()); + exportable = new ElasticMappingExportAdapter(upsertMappingRequestPayload, mappingToUpsert); + exportable.export(); } private ESDsm fetchData() { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java index 6169c0e39..c1f51c9da 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java @@ -4,7 +4,12 @@ import java.time.LocalDateTime; import java.time.LocalTime; import java.time.format.DateTimeParseException; +import java.util.List; +import java.util.Map; +import com.google.common.reflect.TypeToken; +import com.google.gson.Gson; +import com.google.gson.JsonSyntaxException; import org.apache.commons.lang3.StringUtils; public abstract class BaseParser implements Parser { @@ -18,6 +23,8 @@ public Object parse(String value) { result = forBoolean(value); } else if (isDateOrTimeOrDateTime(value)) { result = forDate(value); + } else if (isCollection(value)) { + result = forCollection(value); } return result; } @@ -30,6 +37,8 @@ public Object parse(String value) { protected abstract Object forString(String value); + protected abstract Object forCollection(String value); + boolean isDateOrTimeOrDateTime(String value) { return isDate(value) || isTime(value) || isDateTime(value); } @@ -64,4 +73,15 @@ boolean isDate(String value) { boolean isBoolean(String value) { return value.equalsIgnoreCase("true") || value.equalsIgnoreCase("false"); } + + boolean isCollection(String value) { + List> collection; + try { + collection = new Gson().fromJson(value, new TypeToken>>() { + }.getType()); + } catch (JsonSyntaxException jse) { + collection = null; + } + return collection != null; + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java index 9e316cb4f..b13ee8aa9 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java @@ -1,7 +1,10 @@ package org.broadinstitute.dsm.model.elastic.export.parse; +import com.google.common.reflect.TypeToken; +import com.google.gson.Gson; import org.broadinstitute.dsm.model.elastic.export.generate.MappingGenerator; +import java.util.List; import java.util.Map; public class TypeParser extends BaseParser { @@ -36,4 +39,16 @@ protected Object forDate(String value) { protected Object forString(String value) { return TEXT_KEYWORD_MAPPING; } + + @Override + protected Object forCollection(String value) { + return null; + } + + private Object getNestedType(String value) { + List> list = new Gson().fromJson(value, new TypeToken>>() {}.getType()); + + return null; + } + } \ No newline at end of file diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/ValueParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/ValueParser.java index 10fd5ceb3..dc9948105 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/ValueParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/ValueParser.java @@ -21,4 +21,9 @@ protected Object forDate(String value) { protected Object forString(String value) { return value; } + + @Override + protected Object forCollection(String value) { + return null; + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BulkExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BulkExportFacade.java index 39d1b446e..1988a9c72 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BulkExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BulkExportFacade.java @@ -8,6 +8,7 @@ import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.elasticsearch.action.bulk.BulkRequest; +import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java index 6e90f2100..fd5a54290 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java @@ -1,12 +1,8 @@ package org.broadinstitute.dsm.model.elastic.migration; -import java.lang.reflect.Field; -import java.util.*; -import java.util.stream.Collectors; +import java.util.Map; -import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.OncHistoryDetail; -import org.broadinstitute.dsm.db.structure.TableName; import org.broadinstitute.dsm.statics.ESObjectConstants; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigrator.java new file mode 100644 index 000000000..9dec6757f --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigrator.java @@ -0,0 +1,17 @@ +package org.broadinstitute.dsm.model.elastic.migration; + +import java.util.Map; + +import org.broadinstitute.dsm.db.dao.ddp.tissue.TissueDao; + +public class TissueMigrator extends BaseCollectionMigrator { + + public TissueMigrator(String index, String realm, String object) { + super(index, realm, object); + } + + @Override + protected Map getDataByRealm() { + return (Map) new TissueDao().getTissuesByStudy(realm); + } +} diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserTest.java new file mode 100644 index 000000000..4bd7064f7 --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserTest.java @@ -0,0 +1,27 @@ +package org.broadinstitute.dsm.model.elastic.export.parse; + +import static org.junit.Assert.*; + +import java.util.Arrays; +import java.util.Map; + +import org.junit.Test; + +public class BaseParserTest { + + @Test + public void isCollection() { + String json = "[{\"field1\": \"value1\"}, {\"field2\": \"value2\"}]"; + TypeParser typeParser = new TypeParser(); + boolean isCollection = typeParser.isCollection(json); + assertTrue(isCollection); + } + + @Test + public void isNotCollection() { + String json = "{\"field1\": \"value1\", \"field2\": \"value2\"}"; + TypeParser typeParser = new TypeParser(); + boolean isCollection = typeParser.isCollection(json); + assertFalse(isCollection); + } +} \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigratorTest.java new file mode 100644 index 000000000..d0fa4de93 --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigratorTest.java @@ -0,0 +1,23 @@ +package org.broadinstitute.dsm.model.elastic.migration; + +import static org.junit.Assert.*; + +import java.util.Map; + +import org.broadinstitute.dsm.TestHelper; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TissueMigratorTest { + + @BeforeClass + public static void setUp() { + TestHelper.setupDB(); + } + + @Test + public void getDataByRealm() { + TissueMigrator tissueMigrator = new TissueMigrator("participants_structured.cmi.angio", "angio", "tissues"); + tissueMigrator.export(); + } +} \ No newline at end of file From d7a342a06ed2cf026159c24f4ba9fc2b30515b08 Mon Sep 17 00:00:00 2001 From: Nika Date: Fri, 19 Nov 2021 11:29:32 +0400 Subject: [PATCH 135/405] DDP-7121 | introduce MedicalRecordMappingGenerator --- .../export/generate/GeneratorPayload.java | 2 + .../export/generate/MappingGenerator.java | 1 - .../MedicalRecordMappingGenerator.java | 65 +++++++++++++++++++ .../migration/BaseCollectionMigrator.java | 2 +- 4 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MedicalRecordMappingGenerator.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorPayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorPayload.java index 075389a3c..974221eac 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorPayload.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorPayload.java @@ -20,6 +20,8 @@ public NameValue getNameValue() { return nameValue; } + public Object getValue() { return nameValue.getValue(); } + public int getRecordId() { return recordId; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java index 5af230e0e..961de4e92 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java @@ -20,7 +20,6 @@ public MappingGenerator(Parser parser, GeneratorPayload generatorPayload) { super(parser, generatorPayload); } - @Override public Map generate() { logger.info("preparing mapping to upsert"); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MedicalRecordMappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MedicalRecordMappingGenerator.java new file mode 100644 index 000000000..23e32e99e --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MedicalRecordMappingGenerator.java @@ -0,0 +1,65 @@ +package org.broadinstitute.dsm.model.elastic.export.generate; + +import com.mysql.cj.xdevapi.Column; +import org.broadinstitute.dsm.db.MedicalRecord; +import org.broadinstitute.dsm.db.structure.ColumnName; +import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.model.elastic.export.parse.BaseParser; +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; +import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; +import org.broadinstitute.dsm.model.elastic.migration.BaseCollectionMigrator; +import org.broadinstitute.dsm.model.elastic.migration.MedicalRecordMigrator; + +import javax.lang.model.element.TypeParameterElement; +import java.lang.reflect.Field; +import java.util.HashMap; +import java.util.Map; + +public class MedicalRecordMappingGenerator implements Generator { + + private Parser parser; + private GeneratorPayload generatorPayload; + + public MedicalRecordMappingGenerator(Parser parser, GeneratorPayload generatorPayload) { + this.parser = parser; + this.generatorPayload = generatorPayload; + } + + + @Override + public Map generate() { + + TypeParser typeParser = new TypeParser(); + BaseCollectionMigrator baseCollectionMigrator = new MedicalRecordMigrator("asdad", "Asdad"); + + Field[] declaredFields = MedicalRecord.class.getDeclaredFields(); + + Map map = new HashMap<>(); + + for (Field declaredField : declaredFields) { + + ColumnName annotation = declaredField.getAnnotation(ColumnName.class); + + boolean fieldListType = baseCollectionMigrator.isFieldListType(declaredField); + if (fieldListType) { + try { + Class parameterizedType = Util.getParameterizedType(declaredField.getGenericType()); + + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + } else { + String fieldName = annotation.value(); + String camelCaseFieldName = Util.underscoresToCamelCase(fieldName); + Object type = typeParser.parse(String.valueOf(generatorPayload.getValue())); + map.put(camelCaseFieldName, type); + } + + } + + + + return null; + } + +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java index 4aca1f26b..627408087 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java @@ -64,7 +64,7 @@ private List getListTypeFields(Class clazz) { .collect(Collectors.toList()); } - private boolean isFieldListType(Field field) { + public boolean isFieldListType(Field field) { return List.class.isAssignableFrom(field.getType()); } From ddf1af9d58abb9c46402196e20385a81dbb46d2e Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 19 Nov 2021 15:43:11 +0400 Subject: [PATCH 136/405] DDP-7121 | remove nedless complexity --- .../dsm/model/elastic/Util.java | 3 + .../MedicalRecordMappingGenerator.java | 65 ------------------- .../elastic/export/parse/BaseParser.java | 15 ----- .../elastic/export/parse/TypeParser.java | 11 ---- .../elastic/migration/TissueMigrator.java | 4 +- .../dsm/pubsub/DSMtasksSubscription.java | 3 +- .../elastic/export/parse/BaseParserTest.java | 27 -------- .../elastic/migration/TissueMigratorTest.java | 2 +- 8 files changed, 8 insertions(+), 122 deletions(-) delete mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MedicalRecordMappingGenerator.java delete mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserTest.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index b47277809..c9e34a064 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -110,6 +110,9 @@ public static Map transformObjectToMap(Object obj) { static Map convertToMap(String fieldName, Object fieldValue) { Map finalResult; switch (fieldName) { + case "follow_ups": + finalResult = Map.of(underscoresToCamelCase(fieldName), new Gson().toJson(fieldValue)); + break; case "data": Map objectMap = dynamicFieldsSpecialCase(fieldValue); Map transformedMap = new HashMap<>(); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MedicalRecordMappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MedicalRecordMappingGenerator.java deleted file mode 100644 index 23e32e99e..000000000 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MedicalRecordMappingGenerator.java +++ /dev/null @@ -1,65 +0,0 @@ -package org.broadinstitute.dsm.model.elastic.export.generate; - -import com.mysql.cj.xdevapi.Column; -import org.broadinstitute.dsm.db.MedicalRecord; -import org.broadinstitute.dsm.db.structure.ColumnName; -import org.broadinstitute.dsm.model.elastic.Util; -import org.broadinstitute.dsm.model.elastic.export.parse.BaseParser; -import org.broadinstitute.dsm.model.elastic.export.parse.Parser; -import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; -import org.broadinstitute.dsm.model.elastic.migration.BaseCollectionMigrator; -import org.broadinstitute.dsm.model.elastic.migration.MedicalRecordMigrator; - -import javax.lang.model.element.TypeParameterElement; -import java.lang.reflect.Field; -import java.util.HashMap; -import java.util.Map; - -public class MedicalRecordMappingGenerator implements Generator { - - private Parser parser; - private GeneratorPayload generatorPayload; - - public MedicalRecordMappingGenerator(Parser parser, GeneratorPayload generatorPayload) { - this.parser = parser; - this.generatorPayload = generatorPayload; - } - - - @Override - public Map generate() { - - TypeParser typeParser = new TypeParser(); - BaseCollectionMigrator baseCollectionMigrator = new MedicalRecordMigrator("asdad", "Asdad"); - - Field[] declaredFields = MedicalRecord.class.getDeclaredFields(); - - Map map = new HashMap<>(); - - for (Field declaredField : declaredFields) { - - ColumnName annotation = declaredField.getAnnotation(ColumnName.class); - - boolean fieldListType = baseCollectionMigrator.isFieldListType(declaredField); - if (fieldListType) { - try { - Class parameterizedType = Util.getParameterizedType(declaredField.getGenericType()); - - } catch (ClassNotFoundException e) { - e.printStackTrace(); - } - } else { - String fieldName = annotation.value(); - String camelCaseFieldName = Util.underscoresToCamelCase(fieldName); - Object type = typeParser.parse(String.valueOf(generatorPayload.getValue())); - map.put(camelCaseFieldName, type); - } - - } - - - - return null; - } - -} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java index c1f51c9da..65a66df60 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java @@ -23,8 +23,6 @@ public Object parse(String value) { result = forBoolean(value); } else if (isDateOrTimeOrDateTime(value)) { result = forDate(value); - } else if (isCollection(value)) { - result = forCollection(value); } return result; } @@ -37,8 +35,6 @@ public Object parse(String value) { protected abstract Object forString(String value); - protected abstract Object forCollection(String value); - boolean isDateOrTimeOrDateTime(String value) { return isDate(value) || isTime(value) || isDateTime(value); } @@ -73,15 +69,4 @@ boolean isDate(String value) { boolean isBoolean(String value) { return value.equalsIgnoreCase("true") || value.equalsIgnoreCase("false"); } - - boolean isCollection(String value) { - List> collection; - try { - collection = new Gson().fromJson(value, new TypeToken>>() { - }.getType()); - } catch (JsonSyntaxException jse) { - collection = null; - } - return collection != null; - } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java index b13ee8aa9..153679fd3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java @@ -40,15 +40,4 @@ protected Object forString(String value) { return TEXT_KEYWORD_MAPPING; } - @Override - protected Object forCollection(String value) { - return null; - } - - private Object getNestedType(String value) { - List> list = new Gson().fromJson(value, new TypeToken>>() {}.getType()); - - return null; - } - } \ No newline at end of file diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigrator.java index 9dec6757f..96beb9ca4 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigrator.java @@ -6,8 +6,8 @@ public class TissueMigrator extends BaseCollectionMigrator { - public TissueMigrator(String index, String realm, String object) { - super(index, realm, object); + public TissueMigrator(String index, String realm) { + super(index, realm, "tissues"); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/pubsub/DSMtasksSubscription.java b/src/main/java/org/broadinstitute/dsm/pubsub/DSMtasksSubscription.java index 453a6d76f..e7ed53ffe 100644 --- a/src/main/java/org/broadinstitute/dsm/pubsub/DSMtasksSubscription.java +++ b/src/main/java/org/broadinstitute/dsm/pubsub/DSMtasksSubscription.java @@ -107,7 +107,8 @@ private static void migrateToES(ExportToES.ExportPayload exportPayload) { new OncHistoryDetailsMigrator(index, study), new ParticipantDataMigrator(index, study), new ParticipantMigrator(index, study), - new KitRequestShippingMigrator(index, study)); + new KitRequestShippingMigrator(index, study), + new TissueMigrator(index, study)); exportables.forEach(Exportable::export); }); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserTest.java deleted file mode 100644 index 4bd7064f7..000000000 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserTest.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.broadinstitute.dsm.model.elastic.export.parse; - -import static org.junit.Assert.*; - -import java.util.Arrays; -import java.util.Map; - -import org.junit.Test; - -public class BaseParserTest { - - @Test - public void isCollection() { - String json = "[{\"field1\": \"value1\"}, {\"field2\": \"value2\"}]"; - TypeParser typeParser = new TypeParser(); - boolean isCollection = typeParser.isCollection(json); - assertTrue(isCollection); - } - - @Test - public void isNotCollection() { - String json = "{\"field1\": \"value1\", \"field2\": \"value2\"}"; - TypeParser typeParser = new TypeParser(); - boolean isCollection = typeParser.isCollection(json); - assertFalse(isCollection); - } -} \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigratorTest.java index d0fa4de93..0aef6292d 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigratorTest.java @@ -17,7 +17,7 @@ public static void setUp() { @Test public void getDataByRealm() { - TissueMigrator tissueMigrator = new TissueMigrator("participants_structured.cmi.angio", "angio", "tissues"); + TissueMigrator tissueMigrator = new TissueMigrator("participants_structured.cmi.angio", "angio"); tissueMigrator.export(); } } \ No newline at end of file From 1bf815412167072a869e9042532380f4bec80dba Mon Sep 17 00:00:00 2001 From: Nika Date: Fri, 19 Nov 2021 16:23:18 +0400 Subject: [PATCH 137/405] DDP-7121 | roll back the old version of the code --- .../dsm/db/OncHistoryDetail.java | 1 - .../dsm/model/elastic/Util.java | 24 ------------------- .../elastic/export/parse/ValueParser.java | 5 ---- .../OncHistoryDetailsMigratorTest.java | 19 +++++++++++++++ 4 files changed, 19 insertions(+), 30 deletions(-) create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigratorTest.java diff --git a/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java b/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java index 96833e34a..ead8bbae9 100644 --- a/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java +++ b/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java @@ -171,7 +171,6 @@ public class OncHistoryDetail { private String participantId; - @ColumnName("tissues") private List tissues; public OncHistoryDetail(String oncHistoryDetailId, String medicalRecordId, String datePx, String typePx, diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index c9e34a064..9885f0bc5 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -125,30 +125,6 @@ static Map convertToMap(String fieldName, Object fieldValue) { finalResult = transformedMap; break; default: - if (fieldValue instanceof List) { - List fieldValues = (List) fieldValue; - List> transformedFieldValues = new ArrayList<>(); - for(Object obj: fieldValues) { - Class inside = obj.getClass(); - Field[] declaredFields = inside.getDeclaredFields(); - Map mainMap = new HashMap<>(); - for (Field declaredField : declaredFields) { - declaredField.setAccessible(true); - ColumnName annotation = declaredField.getAnnotation(ColumnName.class); - if (annotation != null) { - String innerFieldName = underscoresToCamelCase(annotation.value()); - try { - if (declaredField.get(obj) != null ) - mainMap.put(innerFieldName, declaredField.get(obj)); - } catch (IllegalAccessException e) { - e.printStackTrace(); - } - } - } - transformedFieldValues.add(mainMap); - } - fieldValue = transformedFieldValues; - } finalResult = Map.of(underscoresToCamelCase(fieldName), fieldValue); break; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/ValueParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/ValueParser.java index dc9948105..10fd5ceb3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/ValueParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/ValueParser.java @@ -21,9 +21,4 @@ protected Object forDate(String value) { protected Object forString(String value) { return value; } - - @Override - protected Object forCollection(String value) { - return null; - } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigratorTest.java new file mode 100644 index 000000000..d79f5a5bd --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigratorTest.java @@ -0,0 +1,19 @@ +package org.broadinstitute.dsm.model.elastic.migration; + +import org.broadinstitute.dsm.TestHelper; +import org.junit.BeforeClass; +import org.junit.Test; + +public class OncHistoryDetailsMigratorTest { + + @BeforeClass + public static void setUp() { + TestHelper.setupDB(); + } + + @Test + public void export() { + OncHistoryDetailsMigrator oncHistoryDetailsMigrator = new OncHistoryDetailsMigrator("participants_structured.cmi.angio", "angio"); + oncHistoryDetailsMigrator.export(); + } +} \ No newline at end of file From ddc746253b94ca728f284816dec38e1bfd1d5009 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 19 Nov 2021 16:54:48 +0400 Subject: [PATCH 138/405] DDP-7121 | add onc history migrator, onc history dao/dto classes --- .../db/dao/ddp/onchistory/OncHistoryDao.java | 75 +++++++++++++++++++ .../dsm/db/dto/onchistory/OncHistoryDto.java | 16 ++++ .../elastic/migration/BaseSingleMigrator.java | 25 +++++++ .../elastic/migration/OncHistoryMigrator.java | 18 +++++ .../migration/ParticipantMigrator.java | 19 +---- .../migration/OncHistoryMigratorTest.java | 22 ++++++ 6 files changed, 157 insertions(+), 18 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/db/dao/ddp/onchistory/OncHistoryDao.java create mode 100644 src/main/java/org/broadinstitute/dsm/db/dto/onchistory/OncHistoryDto.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseSingleMigrator.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryMigrator.java create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryMigratorTest.java diff --git a/src/main/java/org/broadinstitute/dsm/db/dao/ddp/onchistory/OncHistoryDao.java b/src/main/java/org/broadinstitute/dsm/db/dao/ddp/onchistory/OncHistoryDao.java new file mode 100644 index 000000000..cd686f67f --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/db/dao/ddp/onchistory/OncHistoryDao.java @@ -0,0 +1,75 @@ +package org.broadinstitute.dsm.db.dao.ddp.onchistory; + +import static org.broadinstitute.ddp.db.TransactionWrapper.inTransaction; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +import org.broadinstitute.ddp.db.SimpleResult; +import org.broadinstitute.dsm.db.dao.Dao; +import org.broadinstitute.dsm.db.dto.onchistory.OncHistoryDto; +import org.broadinstitute.dsm.statics.DBConstants; + +public class OncHistoryDao implements Dao { + + private static final String SQL_SELECT_ONC_HISTORIES_BY_STUDY = "SELECT p.participant_id, p.ddp_participant_id, " + + "o.onc_history_id, o.created, o.reviewed, " + + "FROM ddp_participant p " + + "LEFT JOIN ddp_instance realm on (p.ddp_instance_id = realm.ddp_instance_id) " + + "LEFT JOIN ddp_onc_history o on (o.participant_id = p.participant_id) " + + "WHERE realm.instance_name = ? "; + public static final String ONC_HISTORY_ID = "onc_history_id"; + public static final String PARTICIPANT_ID = "participant_id"; + public static final String CREATED = "created"; + public static final String REVIEWED = "reviewed"; + + @Override + public int create(OncHistoryDto oncHistoryDto) { + return 0; + } + + @Override + public int delete(int id) { + return 0; + } + + @Override + public Optional get(long id) { + return Optional.empty(); + } + + public Map getOncHistoriesByStudy(String study) { + Map oncHistories = new HashMap<>(); + SimpleResult results = inTransaction((conn) -> { + SimpleResult execResult = new SimpleResult(); + try (PreparedStatement stmt = conn.prepareStatement(SQL_SELECT_ONC_HISTORIES_BY_STUDY)) { + stmt.setString(1, study); + try(ResultSet rs = stmt.executeQuery()) { + while (rs.next()) { + String ddpParticipantId = rs.getString(DBConstants.DDP_PARTICIPANT_ID); + OncHistoryDto oncHistoryDto = new OncHistoryDto( + rs.getInt(ONC_HISTORY_ID), + rs.getInt(PARTICIPANT_ID), + rs.getString(CREATED), + rs.getString(REVIEWED) + ); + oncHistories.put(ddpParticipantId, oncHistoryDto); + } + } + } + catch (SQLException ex) { + execResult.resultException = ex; + } + return execResult; + }); + if (results.resultException != null) { + throw new RuntimeException("Error getting onc histories for " + + study, results.resultException); + } + return oncHistories; + } +} diff --git a/src/main/java/org/broadinstitute/dsm/db/dto/onchistory/OncHistoryDto.java b/src/main/java/org/broadinstitute/dsm/db/dto/onchistory/OncHistoryDto.java new file mode 100644 index 000000000..7eb9a295f --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/db/dto/onchistory/OncHistoryDto.java @@ -0,0 +1,16 @@ +package org.broadinstitute.dsm.db.dto.onchistory; + +public class OncHistoryDto { + + private int oncHistoryId; + private int participantId; + private String created; + private String reviewed; + + public OncHistoryDto(int oncHistoryId, int participantId, String created, String reviewed) { + this.oncHistoryId = oncHistoryId; + this.participantId = participantId; + this.created = created; + this.reviewed = reviewed; + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseSingleMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseSingleMigrator.java new file mode 100644 index 000000000..27c18b287 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseSingleMigrator.java @@ -0,0 +1,25 @@ +package org.broadinstitute.dsm.model.elastic.migration; + +import java.util.Map; + +import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.statics.ESObjectConstants; + +public abstract class BaseSingleMigrator extends BaseMigrator { + + private Map transformedObject; + + public BaseSingleMigrator(String index, String realm, String object) { + super(index, realm, object); + } + + @Override + public Map generate() { + return Map.of(ESObjectConstants.DSM, Map.of(object, transformedObject)); + } + + @Override + protected void transformObject(Object object) { + transformedObject = Util.transformObjectToMap(object); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryMigrator.java new file mode 100644 index 000000000..943bc91af --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryMigrator.java @@ -0,0 +1,18 @@ +package org.broadinstitute.dsm.model.elastic.migration; + +import java.util.Map; + +import org.broadinstitute.dsm.db.dao.ddp.onchistory.OncHistoryDao; +import org.broadinstitute.dsm.statics.ESObjectConstants; + +public class OncHistoryMigrator extends BaseSingleMigrator { + + public OncHistoryMigrator(String index, String realm) { + super(index, realm, ESObjectConstants.ONC_HISTORY); + } + + @Override + protected Map getDataByRealm() { + return (Map) new OncHistoryDao().getOncHistoriesByStudy(realm); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantMigrator.java index 9fa14e974..3adabc6fe 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantMigrator.java @@ -8,31 +8,14 @@ import org.broadinstitute.dsm.model.elastic.export.generate.Generator; import org.broadinstitute.dsm.statics.ESObjectConstants; -public class ParticipantMigrator extends BaseMigrator implements Exportable, Generator { - - private String realm; - protected final BulkExportFacade bulkExportFacade; - private Map transformedObject; +public class ParticipantMigrator extends BaseSingleMigrator implements Exportable, Generator { public ParticipantMigrator(String index, String realm) { super(index, realm, ESObjectConstants.PARTICIPANT); - this.realm = realm; - this.bulkExportFacade = new BulkExportFacade(index); - } - - @Override - public Map generate() { - return Map.of(ESObjectConstants.DSM, Map.of(object, transformedObject)); - } - - @Override - protected void transformObject(Object object) { - transformedObject = Util.transformObjectToMap(object); } @Override protected Map getDataByRealm() { return (Map) Participant.getParticipants(realm); - } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryMigratorTest.java new file mode 100644 index 000000000..74e8b96e3 --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryMigratorTest.java @@ -0,0 +1,22 @@ +package org.broadinstitute.dsm.model.elastic.migration; + +import static org.junit.Assert.*; + +import org.broadinstitute.dsm.TestHelper; +import org.junit.BeforeClass; +import org.junit.Test; + +public class OncHistoryMigratorTest { + + @BeforeClass + public static void setUp() { + TestHelper.setupDB(); + } + + @Test + public void export() { + OncHistoryMigrator oncHistoryMigrator = new OncHistoryMigrator("participants_structured.cmi.angio", "angio"); + oncHistoryMigrator.export(); + } + +} \ No newline at end of file From 0e76dc582cb442a6ff888229aeea773992703aac Mon Sep 17 00:00:00 2001 From: Nika Date: Fri, 19 Nov 2021 18:03:13 +0400 Subject: [PATCH 139/405] DDP-7121 | revisit the older version of code --- .../db/dao/ddp/onchistory/OncHistoryDao.java | 2 +- .../dsm/db/dto/onchistory/OncHistoryDto.java | 18 ++++++++++++++++++ .../broadinstitute/dsm/model/elastic/Util.java | 6 +++++- .../dsm/statics/DBConstants.java | 4 ++++ .../dsm/model/elastic/UtilTest.java | 13 +++++++++++++ 5 files changed, 41 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/dao/ddp/onchistory/OncHistoryDao.java b/src/main/java/org/broadinstitute/dsm/db/dao/ddp/onchistory/OncHistoryDao.java index cd686f67f..2a831c420 100644 --- a/src/main/java/org/broadinstitute/dsm/db/dao/ddp/onchistory/OncHistoryDao.java +++ b/src/main/java/org/broadinstitute/dsm/db/dao/ddp/onchistory/OncHistoryDao.java @@ -17,7 +17,7 @@ public class OncHistoryDao implements Dao { private static final String SQL_SELECT_ONC_HISTORIES_BY_STUDY = "SELECT p.participant_id, p.ddp_participant_id, " + - "o.onc_history_id, o.created, o.reviewed, " + + "o.onc_history_id, o.created, o.reviewed " + "FROM ddp_participant p " + "LEFT JOIN ddp_instance realm on (p.ddp_instance_id = realm.ddp_instance_id) " + "LEFT JOIN ddp_onc_history o on (o.participant_id = p.participant_id) " + diff --git a/src/main/java/org/broadinstitute/dsm/db/dto/onchistory/OncHistoryDto.java b/src/main/java/org/broadinstitute/dsm/db/dto/onchistory/OncHistoryDto.java index 7eb9a295f..6d335a7ca 100644 --- a/src/main/java/org/broadinstitute/dsm/db/dto/onchistory/OncHistoryDto.java +++ b/src/main/java/org/broadinstitute/dsm/db/dto/onchistory/OncHistoryDto.java @@ -1,10 +1,28 @@ package org.broadinstitute.dsm.db.dto.onchistory; +import lombok.Data; +import org.broadinstitute.dsm.db.structure.ColumnName; +import org.broadinstitute.dsm.db.structure.TableName; +import org.broadinstitute.dsm.statics.DBConstants; + +@Data +@TableName( + name = DBConstants.DDP_ONC_HISTORY, + alias = DBConstants.DDP_ONC_HISTORY_ALIAS, + primaryKey = DBConstants.ONC_HISTORY_ID, + columnPrefix = "") public class OncHistoryDto { + @ColumnName(DBConstants.ONC_HISTORY_ID) private int oncHistoryId; + + @ColumnName(DBConstants.PARTICIPANT_ID) private int participantId; + + @ColumnName(DBConstants.ONC_HISTORY_CREATED) private String created; + + @ColumnName(DBConstants.ONC_HISTORY_REVIEWED) private String reviewed; public OncHistoryDto(int oncHistoryId, int participantId, String created, String reviewed) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 9885f0bc5..e74c399c2 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -8,6 +8,8 @@ import java.util.stream.Collectors; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.api.client.json.Json; +import com.google.common.reflect.TypeToken; import com.google.gson.Gson; import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.structure.ColumnName; @@ -111,7 +113,9 @@ static Map convertToMap(String fieldName, Object fieldValue) { Map finalResult; switch (fieldName) { case "follow_ups": - finalResult = Map.of(underscoresToCamelCase(fieldName), new Gson().toJson(fieldValue)); + String str = new Gson().toJson(fieldValue); + List> map = new Gson().fromJson(str, new TypeToken>>(){}.getType()); + finalResult = Map.of(underscoresToCamelCase(fieldName), map); break; case "data": Map objectMap = dynamicFieldsSpecialCase(fieldValue); diff --git a/src/main/java/org/broadinstitute/dsm/statics/DBConstants.java b/src/main/java/org/broadinstitute/dsm/statics/DBConstants.java index 291397e1b..64034f7af 100644 --- a/src/main/java/org/broadinstitute/dsm/statics/DBConstants.java +++ b/src/main/java/org/broadinstitute/dsm/statics/DBConstants.java @@ -132,6 +132,9 @@ public class DBConstants { public static final String CARE_EVOLVE = "CE_order"; public static final String DDP_KIT_TABLE_ABBR = "kit."; + public static final String ONC_HISTORY_ID = "onc_history_id"; + + //carrier service public static final String CARRIER = "carrier"; @@ -483,4 +486,5 @@ public class DBConstants { public static final String LABEL_URL_RETURN = "label_url_return"; + } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java index 995a7f453..b66b27e1a 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java @@ -1,5 +1,6 @@ package org.broadinstitute.dsm.model.elastic; +import static org.broadinstitute.dsm.model.participant.data.ParticipantData.GSON; import static org.junit.Assert.*; import java.lang.reflect.Field; @@ -8,6 +9,7 @@ import java.util.List; import java.util.Map; +import org.broadinstitute.dsm.db.MedicalRecord; import org.broadinstitute.dsm.db.OncHistoryDetail; import org.broadinstitute.dsm.db.Participant; import org.broadinstitute.dsm.db.Tissue; @@ -104,4 +106,15 @@ class MockClass { Assert.fail(); } } + + @Test + public void go() { + + String s = "{followUps: \"[{\"fReceived\":\"2021-11-18\",\"fRequest1\":\"2021-11-18\"},{\"fReceived\":\"2021-11-18\",\"fRequest1\":\"2021-11-18\"}]\"}"; + + MedicalRecord medicalRecord = GSON.fromJson(s, MedicalRecord.class); + + System.out.println(); + + } } \ No newline at end of file From cbd5e2824c088f0e38491199a8e39cc420b4ccc1 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 22 Nov 2021 16:54:26 +0400 Subject: [PATCH 140/405] DDP-7121 | changes for follow up to export as json string --- .../org/broadinstitute/dsm/model/elastic/Util.java | 10 +++++++--- .../dsm/model/elastic/export/parse/BaseParser.java | 12 ++++++++++++ .../dsm/model/elastic/export/parse/TypeParser.java | 2 ++ .../elastic/export/process/CollectionProcessor.java | 3 ++- 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index e74c399c2..ac517f0b3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -35,6 +35,7 @@ public class Util { public static final String DOC = "_doc"; public static final String ID = "id"; private static final Pattern CAMEL_CASE_REGEX = Pattern.compile("(([a-z])+([A-z]))*"); + public static final Gson GSON = new Gson(); public static String getQueryTypeFromId(String id) { String type; @@ -113,9 +114,8 @@ static Map convertToMap(String fieldName, Object fieldValue) { Map finalResult; switch (fieldName) { case "follow_ups": - String str = new Gson().toJson(fieldValue); - List> map = new Gson().fromJson(str, new TypeToken>>(){}.getType()); - finalResult = Map.of(underscoresToCamelCase(fieldName), map); + List> followUps = convertObjectListToMapList(fieldValue); + finalResult = Map.of(underscoresToCamelCase(fieldName), followUps); break; case "data": Map objectMap = dynamicFieldsSpecialCase(fieldValue); @@ -135,6 +135,10 @@ static Map convertToMap(String fieldName, Object fieldValue) { return finalResult; } + public static List> convertObjectListToMapList(Object fieldValue) { + return GSON.fromJson(GSON.toJson(fieldValue), new TypeToken>>(){}.getType()); + } + private static Map dynamicFieldsSpecialCase(Object fieldValue) { Map dynamicMap = Map.of(); if (isJsonInString(fieldValue)) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java index 65a66df60..9f11faeba 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java @@ -23,6 +23,8 @@ public Object parse(String value) { result = forBoolean(value); } else if (isDateOrTimeOrDateTime(value)) { result = forDate(value); + } else if (isCollection(value)) { + result = Map.of("type", "nested"); } return result; } @@ -69,4 +71,14 @@ boolean isDate(String value) { boolean isBoolean(String value) { return value.equalsIgnoreCase("true") || value.equalsIgnoreCase("false"); } + + boolean isCollection(String value) { + boolean isCollection = false; + try { + new Gson().fromJson(value, List.class); + isCollection = true; + } catch (JsonSyntaxException ignored) { + } + return isCollection; + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java index 153679fd3..e4af19882 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java @@ -40,4 +40,6 @@ protected Object forString(String value) { return TEXT_KEYWORD_MAPPING; } + + } \ No newline at end of file diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java index abd03bae5..68140a239 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java @@ -55,7 +55,7 @@ private List> extractDataByReflection() { private List> getRecordsByField(Field field) { try { - return (List>) field.get(esDsm); + return Util.convertObjectListToMapList(field.get(esDsm)); } catch (IllegalAccessException iae) { throw new RuntimeException("error occurred while attempting to get data from ESDsm", iae); } @@ -75,6 +75,7 @@ private void addNewRecordTo(List> fetchedRecords) { } private boolean isExistingRecord(Map eachRecord) { + if (!eachRecord.containsKey(Util.ID)) return false; return (double) eachRecord.get(Util.ID) == (double) generatorPayload.getRecordId(); } From c3f1866044f912933c605a7cf72f7f9744e4b14e Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 22 Nov 2021 17:58:19 +0400 Subject: [PATCH 141/405] DDP-7121 | introduce Collection and Single mapping generators --- .../export/generate/BaseGenerator.java | 28 ++++++++------ .../generate/CollectionMappingGenerator.java | 29 ++++++++++++++ .../export/generate/MappingGenerator.java | 38 +++++++++---------- .../generate/SingleMappingGenerator.java | 23 +++++++++++ 4 files changed, 88 insertions(+), 30 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java index a34fa1613..e46738d60 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java @@ -77,23 +77,29 @@ protected Object getFieldWithElement() { return fieldElementMap; } + protected abstract Object getElement(Object type); + protected abstract Object getElementWithId(Object element); protected abstract Map getElement(Object element); - protected Object constructByPropertyType() { - Object constructedObject; - if (getOuterPropertyByAlias().isCollection()) { - constructedObject = constructCollection(); - } else { - constructedObject = constructSingleElement(); - } - return constructedObject; - } +// protected Object constructByPropertyType() { +// Object constructedObject; +// if (getOuterPropertyByAlias().isCollection()) { +// constructedObject = constructCollection(); +// } else { +// constructedObject = constructSingleElement(); +// } +// return constructedObject; +// +// // construct(); +// } + +// protected abstract Object constructSingleElement(); - protected abstract Object constructSingleElement(); +// protected abstract Object constructCollection(); - protected abstract Object constructCollection(); + protected abstract Object construct(); public static class PropertyInfo { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java new file mode 100644 index 000000000..efd28a618 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java @@ -0,0 +1,29 @@ +package org.broadinstitute.dsm.model.elastic.export.generate; + +import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; + +import java.util.HashMap; +import java.util.Map; + +public class CollectionMappingGenerator extends MappingGenerator { + + public CollectionMappingGenerator(Parser parser, GeneratorPayload generatorPayload) { + super(parser, generatorPayload); + } + + @Override + protected Object getElement() { + return Map.of( + Util.ID, Map.of(TYPE, TYPE_KEYWORD), + Util.underscoresToCamelCase(getDBElement().getColumnName()), type + ); + } + + @Override + protected Object construct() { + return new HashMap<>(Map.of(TYPE, NESTED, PROPERTIES, collect())); + } + + +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java index 961de4e92..728de0ca7 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java @@ -8,7 +8,7 @@ import java.util.HashMap; import java.util.Map; -public class MappingGenerator extends BaseGenerator implements Merger { +abstract public class MappingGenerator extends BaseGenerator implements Merger { private static final Logger logger = LoggerFactory.getLogger(MappingGenerator.class); @@ -32,7 +32,7 @@ public Map generate() { } private Map buildMappedField() { - return (Map) constructByPropertyType(); + return (Map) construct(); } @Override @@ -48,31 +48,31 @@ protected Map parseJson() { @Override protected Object parseSingleElement() { - return getFieldWithElement(); - } - - @Override - protected Map getElementWithId(Object type) { - return Map.of( - Util.ID, Map.of(TYPE, TYPE_KEYWORD), - Util.underscoresToCamelCase(getDBElement().getColumnName()), type - ); + return getElement(); } +// +// @Override +// protected Map getElementWithId(Object type) { +// return Map.of( +// Util.ID, Map.of(TYPE, TYPE_KEYWORD), +// Util.underscoresToCamelCase(getDBElement().getColumnName()), type +// ); +// } @Override protected Map getElement(Object type) { return Map.of(Util.underscoresToCamelCase(getDBElement().getColumnName()), type); } - @Override - protected Object constructSingleElement() { - return new HashMap<>(Map.of(PROPERTIES, collect())); - } +// @Override +// protected Object constructSingleElement() { +// return new HashMap<>(Map.of(PROPERTIES, collect())); +// } - @Override - protected Object constructCollection() { - return new HashMap<>(Map.of(TYPE, NESTED, PROPERTIES, collect())); - } +// @Override +// protected Object constructCollection() { +// return new HashMap<>(Map.of(TYPE, NESTED, PROPERTIES, collect())); +// } @Override public Map merge(Map base, Map toMerge) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java new file mode 100644 index 000000000..4e9ffc90f --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java @@ -0,0 +1,23 @@ +package org.broadinstitute.dsm.model.elastic.export.generate; + +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; + +import java.util.HashMap; +import java.util.Map; + +public class SingleMappingGenerator extends MappingGenerator { + + public SingleMappingGenerator(Parser parser, GeneratorPayload generatorPayload) { + super(parser, generatorPayload); + } + + @Override + protected Object getElement() { + return null; + } + + @Override + protected Object construct() { + return new HashMap<>(Map.of(PROPERTIES, collect())); + } +} From 45c4225fd99855ff0d3b68169706fa399dc0cbc0 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 22 Nov 2021 18:44:44 +0400 Subject: [PATCH 142/405] DDP-7121 | add single source generator and collection source generator --- .../export/generate/BaseGenerator.java | 37 +++---------------- .../generate/CollectionMappingGenerator.java | 4 +- .../generate/CollectionSourceGenerator.java | 25 +++++++++++++ .../export/generate/MappingGenerator.java | 34 +---------------- .../generate/SingleMappingGenerator.java | 7 ++-- .../generate/SingleSourceGenerator.java | 25 +++++++++++++ .../export/generate/SourceGenerator.java | 11 +----- 7 files changed, 63 insertions(+), 80 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleSourceGenerator.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java index e46738d60..efdcf4f12 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java @@ -1,5 +1,6 @@ package org.broadinstitute.dsm.model.elastic.export.generate; +import java.util.Collection; import java.util.Map; import java.util.Objects; @@ -64,42 +65,14 @@ protected Map parseJsonToMapFromValue() { return GSON.fromJson(String.valueOf(getNameValue().getValue()), Map.class); } - protected abstract T parseSingleElement(); - - protected Object getFieldWithElement() { - Object fieldElementMap; + protected Object parseSingleElement() { Object element = parser.parse(String.valueOf(getNameValue().getValue())); - if (getOuterPropertyByAlias().isCollection()) { - fieldElementMap = getElementWithId(element); - } else { - fieldElementMap = getElement(element); - } - return fieldElementMap; - } + return getElement(element); + }; protected abstract Object getElement(Object type); - protected abstract Object getElementWithId(Object element); - - protected abstract Map getElement(Object element); - -// protected Object constructByPropertyType() { -// Object constructedObject; -// if (getOuterPropertyByAlias().isCollection()) { -// constructedObject = constructCollection(); -// } else { -// constructedObject = constructSingleElement(); -// } -// return constructedObject; -// -// // construct(); -// } - -// protected abstract Object constructSingleElement(); - -// protected abstract Object constructCollection(); - - protected abstract Object construct(); + protected abstract Map construct(); public static class PropertyInfo { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java index efd28a618..b8e497149 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java @@ -13,7 +13,7 @@ public CollectionMappingGenerator(Parser parser, GeneratorPayload generatorPaylo } @Override - protected Object getElement() { + protected Map getElement(Object type) { return Map.of( Util.ID, Map.of(TYPE, TYPE_KEYWORD), Util.underscoresToCamelCase(getDBElement().getColumnName()), type @@ -21,7 +21,7 @@ protected Object getElement() { } @Override - protected Object construct() { + protected Map construct() { return new HashMap<>(Map.of(TYPE, NESTED, PROPERTIES, collect())); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java new file mode 100644 index 000000000..6cef657a8 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java @@ -0,0 +1,25 @@ +package org.broadinstitute.dsm.model.elastic.export.generate; + +import java.util.List; +import java.util.Map; + +import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; + +public class CollectionSourceGenerator extends SourceGenerator{ + public CollectionSourceGenerator(Parser parser, + GeneratorPayload generatorPayload) { + super(parser, generatorPayload); + } + + @Override + protected Object getElement(Object element) { + return List.of(Map.of(Util.underscoresToCamelCase(getDBElement().getColumnName()), element, + Util.ID, generatorPayload.getRecordId()));; + } + + @Override + protected Map construct() { + return null; + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java index 728de0ca7..24ccc172b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java @@ -24,16 +24,12 @@ public MappingGenerator(Parser parser, GeneratorPayload generatorPayload) { public Map generate() { logger.info("preparing mapping to upsert"); String propertyName = getOuterPropertyByAlias().getPropertyName(); - Map mappedField = buildMappedField(); - Map objectLevel = Map.of(propertyName, mappedField); + Map objectLevel = Map.of(propertyName, construct()); Map dsmLevelProperties = Map.of(PROPERTIES, objectLevel); Map> dsmLevel = Map.of(DSM_OBJECT, dsmLevelProperties); return Map.of(PROPERTIES, dsmLevel); } - private Map buildMappedField() { - return (Map) construct(); - } @Override protected Map parseJson() { @@ -46,34 +42,6 @@ protected Map parseJson() { return resultMap; } - @Override - protected Object parseSingleElement() { - return getElement(); - } -// -// @Override -// protected Map getElementWithId(Object type) { -// return Map.of( -// Util.ID, Map.of(TYPE, TYPE_KEYWORD), -// Util.underscoresToCamelCase(getDBElement().getColumnName()), type -// ); -// } - - @Override - protected Map getElement(Object type) { - return Map.of(Util.underscoresToCamelCase(getDBElement().getColumnName()), type); - } - -// @Override -// protected Object constructSingleElement() { -// return new HashMap<>(Map.of(PROPERTIES, collect())); -// } - -// @Override -// protected Object constructCollection() { -// return new HashMap<>(Map.of(TYPE, NESTED, PROPERTIES, collect())); -// } - @Override public Map merge(Map base, Map toMerge) { base.putIfAbsent(PROPERTIES, toMerge.get(PROPERTIES)); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java index 4e9ffc90f..5f290afe3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java @@ -1,5 +1,6 @@ package org.broadinstitute.dsm.model.elastic.export.generate; +import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import java.util.HashMap; @@ -12,12 +13,12 @@ public SingleMappingGenerator(Parser parser, GeneratorPayload generatorPayload) } @Override - protected Object getElement() { - return null; + protected Map getElement(Object type) { + return Map.of(Util.underscoresToCamelCase(getDBElement().getColumnName()), type); } @Override - protected Object construct() { + protected Map construct() { return new HashMap<>(Map.of(PROPERTIES, collect())); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleSourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleSourceGenerator.java new file mode 100644 index 000000000..04d52ca10 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleSourceGenerator.java @@ -0,0 +1,25 @@ +package org.broadinstitute.dsm.model.elastic.export.generate; + +import java.util.Map; + +import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; + +public class SingleSourceGenerator extends SourceGenerator { + public SingleSourceGenerator(Parser parser, + GeneratorPayload generatorPayload) { + super(parser, generatorPayload); + } + + @Override + protected Map construct() { + return null; + } + + @Override + protected Map getElement(Object element) { + logger.info("Constructing single field with value"); + return Map.of(Util.underscoresToCamelCase(getDBElement().getColumnName()), element); + } +} + diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java index b1be50015..2acb8fef6 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java @@ -9,7 +9,7 @@ import java.util.List; import java.util.Map; -public class SourceGenerator extends BaseGenerator { +abstract public class SourceGenerator extends BaseGenerator { private static final Logger logger = LoggerFactory.getLogger(SourceGenerator.class); @@ -49,21 +49,12 @@ protected Object parseSingleElement() { return getFieldWithElement(); } - @Override - protected Object getElementWithId(Object element) { - return buildCollectionWithId(element); - } private List> buildCollectionWithId(Object element) { return List.of(Map.of(Util.underscoresToCamelCase(getDBElement().getColumnName()), element, Util.ID, generatorPayload.getRecordId())); } - @Override - protected Map getElement(Object element) { - return buildSingleFieldWithValue(element); - } - private Map buildSingleFieldWithValue(Object element) { logger.info("Constructing single field with value"); return Map.of(Util.underscoresToCamelCase(getDBElement().getColumnName()), element); From 0b45ff8f5ac71430daad91957868189fa5dacbbd Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 22 Nov 2021 19:09:55 +0400 Subject: [PATCH 143/405] DDP-7121 | refactor further, start writing tests for new code --- .../export/generate/BaseGenerator.java | 2 +- .../generate/CollectionSourceGenerator.java | 23 ++++++++---- .../generate/SingleSourceGenerator.java | 9 +++-- .../export/generate/SourceGenerator.java | 36 ++----------------- .../export/generate/MappingGeneratorTest.java | 17 +++++++++ 5 files changed, 44 insertions(+), 43 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java index efdcf4f12..3ea6efc7e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java @@ -72,7 +72,7 @@ protected Object parseSingleElement() { protected abstract Object getElement(Object type); - protected abstract Map construct(); + protected abstract Object construct(); public static class PropertyInfo { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java index 6cef657a8..0eb937b85 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java @@ -1,25 +1,36 @@ package org.broadinstitute.dsm.model.elastic.export.generate; +import java.util.HashMap; import java.util.List; import java.util.Map; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CollectionSourceGenerator extends SourceGenerator { + + private static final Logger logger = LoggerFactory.getLogger(CollectionSourceGenerator.class); -public class CollectionSourceGenerator extends SourceGenerator{ public CollectionSourceGenerator(Parser parser, GeneratorPayload generatorPayload) { super(parser, generatorPayload); } @Override - protected Object getElement(Object element) { - return List.of(Map.of(Util.underscoresToCamelCase(getDBElement().getColumnName()), element, - Util.ID, generatorPayload.getRecordId()));; + protected Object construct() { + logger.info("Constructing nested data"); + Map collectionMap = new HashMap<>(); + collectionMap.put(Util.ID, generatorPayload.getRecordId()); + Map mapWithParsedObjects = parseJsonValuesToObject(); + collectionMap.putAll(mapWithParsedObjects); + return List.of(collectionMap); } @Override - protected Map construct() { - return null; + protected Object getElement(Object element) { + return List.of(Map.of(Util.underscoresToCamelCase(getDBElement().getColumnName()), element, + Util.ID, generatorPayload.getRecordId())); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleSourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleSourceGenerator.java index 04d52ca10..ecb1ac420 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleSourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleSourceGenerator.java @@ -4,16 +4,21 @@ import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class SingleSourceGenerator extends SourceGenerator { + + private static final Logger logger = LoggerFactory.getLogger(SingleSourceGenerator.class); + public SingleSourceGenerator(Parser parser, GeneratorPayload generatorPayload) { super(parser, generatorPayload); } @Override - protected Map construct() { - return null; + protected Object construct() { + return parseJsonValuesToObject(); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java index 2acb8fef6..166155ec5 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java @@ -31,10 +31,10 @@ private Map buildPropertyLevelWithData(Object dataToExport) { @Override protected Object parseJson() { - return constructByPropertyType(); + return construct(); } - private Map parseJsonValuesToObject() { + protected Map parseJsonValuesToObject() { logger.info("Converting JSON values to Map"); Map dynamicFieldValues = parseJsonToMapFromValue(); Map transformedMap = new HashMap<>(); @@ -44,36 +44,4 @@ private Map parseJsonValuesToObject() { return transformedMap; } - @Override - protected Object parseSingleElement() { - return getFieldWithElement(); - } - - - private List> buildCollectionWithId(Object element) { - return List.of(Map.of(Util.underscoresToCamelCase(getDBElement().getColumnName()), element, - Util.ID, generatorPayload.getRecordId())); - } - - private Map buildSingleFieldWithValue(Object element) { - logger.info("Constructing single field with value"); - return Map.of(Util.underscoresToCamelCase(getDBElement().getColumnName()), element); - } - - @Override - protected Object constructSingleElement() { - return parseJsonValuesToObject(); - } - - @Override - protected Object constructCollection() { - logger.info("Constructing nested data"); - Map collectionMap = new HashMap<>(); - collectionMap.put(Util.ID, generatorPayload.getRecordId()); - Map mapWithParsedObjects = parseJsonValuesToObject(); - collectionMap.putAll(mapWithParsedObjects); - return List.of(collectionMap); - } - - } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java index 36b560f67..e3c0beee1 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java @@ -164,6 +164,23 @@ private Map getMedicalRecordProperty(Map objectMap) { } + private static class TestSingleMappingGenerator extends SingleMappingGenerator { + + public TestSingleMappingGenerator(Parser parser, GeneratorPayload generatorPayload) { + super(parser, generatorPayload); + } + + @Override + protected DBElement getDBElement() { + return TestPatchUtil.getColumnNameMap().get(getNameValue().getName()); + } + + public static TestSingleMappingGenerator of(GeneratorPayload generatorPayload) { + return new TestSingleMappingGenerator(new TypeParser(), generatorPayload); + } + + } + private static class TestMappingGenerator extends MappingGenerator { public TestMappingGenerator(Parser typeParser, GeneratorPayload generatorPayload) { From 07ee9976d30157400b16a300e9b20d4cdfaa3223 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 23 Nov 2021 10:44:23 +0400 Subject: [PATCH 144/405] DDP-7121 | separate single/collection generators, add exportable factory and mapping factory implementation --- .../model/elastic/export/ExportFacade.java | 12 +++-- .../elastic/export/ExportFacadePayload.java | 12 ++++- .../elastic/export/ExportableFactory.java | 9 ++++ .../export/MappingExporterFactory.java | 20 ++++++++ .../elastic/export/NullObjectExporter.java | 9 ++++ .../export/generate/BaseGenerator.java | 11 ++++ .../export/generate/GeneratorPayload.java | 2 + .../MedicalRecordMappingGenerator.java | 21 ++++++++ .../export/process/CollectionProcessor.java | 8 +-- .../migration/ParticipantDataMigrator.java | 3 +- .../export/generate/MappingGeneratorTest.java | 50 +++++++++---------- .../export/generate/SourceGeneratorTest.java | 15 +++--- .../process/CollectionProcessorTest.java | 3 +- 13 files changed, 130 insertions(+), 45 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportableFactory.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingExporterFactory.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/NullObjectExporter.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MedicalRecordMappingGenerator.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 1641d1e1f..52f15db3a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -43,9 +43,12 @@ public void export() { } private void upsertMapping() { - generator = new MappingGenerator(new TypeParser(), exportFacadePayload.getGeneratorPayload()); +// generator = new MappingGenerator(new TypeParser(), exportFacadePayload.getGeneratorPayload()); + generator = null; Map mappingToUpsert = generator.generate(); UpsertMappingRequestPayload upsertMappingRequestPayload = new UpsertMappingRequestPayload(exportFacadePayload.getIndex()); + BaseGenerator.PropertyInfo propertyInfo = getPropertyInfo(); + propertyInfo.setFieldName(Util.underscoresToCamelCase(exportFacadePayload.getFieldName())); exportable = new ElasticMappingExportAdapter(upsertMappingRequestPayload, mappingToUpsert); exportable.export(); } @@ -60,12 +63,13 @@ private ESDsm fetchData() { private Map processData(ESDsm esDsm) { BaseGenerator.PropertyInfo propertyInfo = getPropertyInfo(); ValueParser valueParser = new ValueParser(); - SourceGenerator sourceGenerator = new SourceGenerator(valueParser, exportFacadePayload.getGeneratorPayload()); +// SourceGenerator sourceGenerator = new SourceGenerator(valueParser, exportFacadePayload.getGeneratorPayload()); + SourceGenerator sourceGenerator = null; this.generator = sourceGenerator; Map dataToReturn = generator.generate(); logger.info("Processing ES participant data"); if (propertyInfo.isCollection()) { - processor = new CollectionProcessor(esDsm, propertyInfo.getPropertyName(), exportFacadePayload.getGeneratorPayload(), + processor = new CollectionProcessor(esDsm, propertyInfo.getPropertyName(), exportFacadePayload.getRecordId(), sourceGenerator); List> processedData = processor.process(); if (!processedData.isEmpty()) { @@ -77,7 +81,7 @@ private Map processData(ESDsm esDsm) { } private BaseGenerator.PropertyInfo getPropertyInfo() { - DBElement dbElement = PatchUtil.getColumnNameMap().get(exportFacadePayload.getGeneratorPayload().getNameValue().getName()); + DBElement dbElement = PatchUtil.getColumnNameMap().get(exportFacadePayload.getFieldName()); return Util.TABLE_ALIAS_MAPPINGS.get(dbElement.getTableAlias()); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java index 55f50ef78..eba81b099 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java @@ -2,7 +2,10 @@ import java.util.Objects; +import org.broadinstitute.dsm.db.structure.DBElement; +import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; +import org.broadinstitute.dsm.util.PatchUtil; public class ExportFacadePayload { private String index; @@ -23,11 +26,16 @@ public String getDocId() { return docId; } - public GeneratorPayload getGeneratorPayload() { - return generatorPayload; + public int getRecordId() { + return generatorPayload.getRecordId(); } public void setDocId(String docId) { this.docId = docId; } + + public String getFieldName() { + DBElement dbElement = PatchUtil.getColumnNameMap().get(generatorPayload.getName()); + return dbElement.getColumnName(); + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportableFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportableFactory.java new file mode 100644 index 000000000..5fbdeb257 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportableFactory.java @@ -0,0 +1,9 @@ +package org.broadinstitute.dsm.model.elastic.export; + +import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; + +public interface ExportableFactory { + + Exportable make(BaseGenerator.PropertyInfo propertyInfo); + +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingExporterFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingExporterFactory.java new file mode 100644 index 000000000..595c2e5e7 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingExporterFactory.java @@ -0,0 +1,20 @@ +package org.broadinstitute.dsm.model.elastic.export; + +import java.util.Objects; + +import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; + +public class MappingExporterFactory implements ExportableFactory { + + + @Override + public Exportable make(BaseGenerator.PropertyInfo propertyInfo) { + BaseExporter exporter = new NullObjectExporter(); + if (Objects.requireNonNull(propertyInfo).isCollection()) { + if (propertyInfo.getFieldName()) + } else { + + } + return exporter; + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/NullObjectExporter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/NullObjectExporter.java new file mode 100644 index 000000000..4141513a0 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/NullObjectExporter.java @@ -0,0 +1,9 @@ +package org.broadinstitute.dsm.model.elastic.export; + +public class NullObjectExporter extends BaseExporter { + + @Override + public void export() { + + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java index 3ea6efc7e..2efc66e24 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java @@ -6,6 +6,7 @@ import com.google.gson.Gson; import com.google.gson.JsonSyntaxException; +import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.elastic.Util; @@ -78,6 +79,7 @@ public static class PropertyInfo { private String propertyName; private boolean isCollection; + private String fieldName; public PropertyInfo(String propertyName, boolean isCollection) { this.propertyName = Objects.requireNonNull(propertyName); @@ -95,6 +97,15 @@ public String getPropertyName() { public boolean isCollection() { return isCollection; } + + public void setFieldName(String fieldName) { + this.fieldName = Objects.requireNonNull(fieldName); + } + + public String getFieldName() { + if (StringUtils.isBlank(this.fieldName)) this.fieldName = ""; + return this.fieldName; + } } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorPayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorPayload.java index 974221eac..9023e3809 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorPayload.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorPayload.java @@ -20,6 +20,8 @@ public NameValue getNameValue() { return nameValue; } + public String getName() {return nameValue.getName();} + public Object getValue() { return nameValue.getValue(); } public int getRecordId() { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MedicalRecordMappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MedicalRecordMappingGenerator.java new file mode 100644 index 000000000..1f89f0b0f --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MedicalRecordMappingGenerator.java @@ -0,0 +1,21 @@ +package org.broadinstitute.dsm.model.elastic.export.generate; + +import java.util.Map; + +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; + +public class MedicalRecordMappingGenerator extends CollectionMappingGenerator{ + public MedicalRecordMappingGenerator(Parser parser, + GeneratorPayload generatorPayload) { + super(parser, generatorPayload); + } + + @Override + public Object collect() { + Map collect = (Map) super.collect(); + if ("followUps".equals(generatorPayload.getName())) { + collect.remove("followUps"); + } + return collect; + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java index 68140a239..bfbccb88a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java @@ -22,15 +22,15 @@ public class CollectionProcessor implements Processor { private ESDsm esDsm; private String propertyName; - private GeneratorPayload generatorPayload; + private int recordId; private Collector collector; private final Predicate isFieldMatchProperty = field -> propertyName.equals(field.getName()); - public CollectionProcessor(ESDsm esDsm, String propertyName, GeneratorPayload generatorPayload, Collector collector) { + public CollectionProcessor(ESDsm esDsm, String propertyName, int recordId, Collector collector) { this.esDsm = Objects.requireNonNull(esDsm); this.propertyName = Objects.requireNonNull(propertyName); - this.generatorPayload = Objects.requireNonNull(generatorPayload); + this.recordId = recordId; this.collector = collector; } @@ -76,7 +76,7 @@ private void addNewRecordTo(List> fetchedRecords) { private boolean isExistingRecord(Map eachRecord) { if (!eachRecord.containsKey(Util.ID)) return false; - return (double) eachRecord.get(Util.ID) == (double) generatorPayload.getRecordId(); + return (double) eachRecord.get(Util.ID) == (double) recordId; } private void updateExistingRecord(Map eachRecord) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java index 3f6ebdd50..3a953194e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java @@ -51,7 +51,8 @@ private void exportMapping() { for (Object jsonData : participantDatas) { NameValue nameValue = new NameValue(DATA_WITH_ALIAS, dataField.get(jsonData)); GeneratorPayload generatorPayload = new GeneratorPayload(nameValue); - MappingGenerator mappingGenerator = new MappingGenerator(typeParser, generatorPayload); +// MappingGenerator mappingGenerator = new MappingGenerator(typeParser, generatorPayload); + MappingGenerator mappingGenerator = null; mapping = mappingGenerator.merge(mapping, mappingGenerator.generate()); } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java index e3c0beee1..7a6d94bfd 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java @@ -23,7 +23,7 @@ public void generateTextType() { new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "value"), 0 ); - Map objectMap = TestMappingGenerator.of(generatorPayload).generate(); + Map objectMap = TestCollectionMappingGenerator.of(generatorPayload).generate(); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), BaseGenerator.PROPERTIES); String type = extractDeepestLeveleValue(objectMap); Assert.assertEquals("text", type); @@ -35,7 +35,7 @@ public void generateTextTypeWithFields() { new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "value"), 0 ); - Map objectMap = TestMappingGenerator.of(generatorPayload).generate(); + Map objectMap = TestCollectionMappingGenerator.of(generatorPayload).generate(); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), BaseGenerator.PROPERTIES); String type = extractKeywordType(objectMap); Assert.assertEquals("keyword", type); @@ -47,7 +47,7 @@ public void generateBooleanType() { new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "true"), 0 ); - Map objectMap = TestMappingGenerator.of(generatorPayload).generate(); + Map objectMap = TestCollectionMappingGenerator.of(generatorPayload).generate(); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), BaseGenerator.PROPERTIES); String type = extractDeepestLeveleValue(objectMap); Assert.assertEquals("boolean", type); @@ -59,7 +59,7 @@ public void generateDateType() { new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "2021-10-30"), 0 ); - Map objectMap = TestMappingGenerator.of(generatorPayload).generate(); + Map objectMap = TestCollectionMappingGenerator.of(generatorPayload).generate(); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), BaseGenerator.PROPERTIES); String type = extractDeepestLeveleValue(objectMap); Assert.assertEquals("date", type); @@ -71,33 +71,21 @@ public void generateNestedType() { new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "2021-10-30"), 100 ); - Map objectMap = TestMappingGenerator.of(generatorPayload).generate(); + Map objectMap = TestCollectionMappingGenerator.of(generatorPayload).generate(); Assert.assertEquals(MappingGenerator.NESTED, getMedicalRecordProperty(objectMap).get(MappingGenerator.TYPE)); } - @Test - public void getFieldWithTypeCollectionTrue() { - GeneratorPayload generatorPayload = new GeneratorPayload( - new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "2021-10-30"), - 100 - ); - TestMappingGenerator generator = TestMappingGenerator.of(generatorPayload); - generator.getOuterPropertyByAlias().setIsCollection(true); - Object fieldWithType = generator.getFieldWithElement(); - Assert.assertTrue(((Map) fieldWithType).containsKey(Util.ID)); - } - @Test public void generateMapping() { GeneratorPayload generatorPayload = new GeneratorPayload( new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "2021-10-30"), 100 ); - MappingGenerator generator = new TestMappingGenerator(new TypeParser(), generatorPayload); + MappingGenerator generator = TestCollectionMappingGenerator.of(generatorPayload); Map resultMap = generator.generate(); Map dsmLevelProperty = Map.of(generator.getOuterPropertyByAlias().getPropertyName(), Map.of( MappingGenerator.TYPE, MappingGenerator.NESTED, - MappingGenerator.PROPERTIES, Map.of(TestPatchUtil.MEDICAL_RECORD_COLUMN, Map.of(MappingGenerator.TYPE, "date"), + MappingGenerator.PROPERTIES, Map.of(Util.underscoresToCamelCase(TestPatchUtil.MEDICAL_RECORD_COLUMN), Map.of(MappingGenerator.TYPE, "date"), Util.ID, Map.of(MappingGenerator.TYPE, MappingGenerator.TYPE_KEYWORD) ))); Map dsmLevelProperties = Map.of(MappingGenerator.PROPERTIES, dsmLevelProperty); @@ -111,12 +99,12 @@ public void merge() { NameValue nameValue = new NameValue("d.data", "{\"DDP_INSTANCE\": \"TEST\"}"); GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, 0); TypeParser parser = new TypeParser(); - MappingGenerator mappingGenerator = new MappingGenerator(parser, generatorPayload); + MappingGenerator mappingGenerator = new CollectionMappingGenerator(parser, generatorPayload); Map base = new HashMap<>(); base = mappingGenerator.merge(base, mappingGenerator.generate()); NameValue nameValue2 = new NameValue("d.data", "{\"DDP_INSTANCE1\": \"TEST1\"}"); GeneratorPayload generatorPayload2 = new GeneratorPayload(nameValue2, 0); - mappingGenerator = new MappingGenerator(parser, generatorPayload2); + mappingGenerator = new CollectionMappingGenerator(parser, generatorPayload2); base = mappingGenerator.merge(base, mappingGenerator.generate()); Object value = ((Map) ((Map) ((Map) ((Map) ((Map) base .get(MappingGenerator.PROPERTIES)) @@ -124,10 +112,20 @@ public void merge() { .get(MappingGenerator.PROPERTIES)) .get(mappingGenerator.getOuterPropertyByAlias().getPropertyName())) .get(MappingGenerator.PROPERTIES)) - .get("DDP_INSTANCE1"); + .get(Util.underscoresToCamelCase("DDP_INSTANCE1")); Assert.assertFalse(Objects.isNull(value)); } + @Test + public void removeFollowUpsFromConstruct() { + NameValue nameValue = new NameValue("m.followUps", "TEST"); + GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, 0); + TypeParser parser = new TypeParser(); + MappingGenerator mappingGenerator = new MedicalRecordMappingGenerator(parser, generatorPayload); + Map objectMap = mappingGenerator.generate(); + + } + private String extractDeepestLeveleValue(Map objectMap) { return (String) ((Map) @@ -181,9 +179,9 @@ public static TestSingleMappingGenerator of(GeneratorPayload generatorPayload) { } - private static class TestMappingGenerator extends MappingGenerator { + private static class TestCollectionMappingGenerator extends CollectionMappingGenerator { - public TestMappingGenerator(Parser typeParser, GeneratorPayload generatorPayload) { + public TestCollectionMappingGenerator(Parser typeParser, GeneratorPayload generatorPayload) { super(typeParser, generatorPayload); } @@ -192,8 +190,8 @@ protected DBElement getDBElement() { return TestPatchUtil.getColumnNameMap().get(getNameValue().getName()); } - public static TestMappingGenerator of(GeneratorPayload generatorPayload) { - return new TestMappingGenerator(new TypeParser(), generatorPayload); + public static TestCollectionMappingGenerator of(GeneratorPayload generatorPayload) { + return new TestCollectionMappingGenerator(new TypeParser(), generatorPayload); } } } \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorTest.java index fd3a1f536..3bdd573bb 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorTest.java @@ -6,6 +6,7 @@ import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; +import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import org.broadinstitute.dsm.model.elastic.export.TestPatchUtil; import org.broadinstitute.dsm.model.elastic.export.parse.ValueParser; @@ -23,9 +24,9 @@ public void generateCollection() { Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), SourceGenerator.DSM_OBJECT); List> medicalRecords = (List) ((Map) objectMap.get(SourceGenerator.DSM_OBJECT)).get("medicalRecords"); Optional> first = medicalRecords.stream() - .filter(i -> i.get(TestPatchUtil.MEDICAL_RECORD_COLUMN) != null) + .filter(i -> i.get(Util.underscoresToCamelCase(TestPatchUtil.MEDICAL_RECORD_COLUMN)) != null) .findFirst(); - first.ifPresentOrElse(val -> Assert.assertEquals("value", val.get(TestPatchUtil.MEDICAL_RECORD_COLUMN)), Assert::fail); + first.ifPresentOrElse(val -> Assert.assertEquals("value", val.get(Util.underscoresToCamelCase(TestPatchUtil.MEDICAL_RECORD_COLUMN))), Assert::fail); } @Test @@ -37,9 +38,9 @@ public void generateNumeric() { Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), SourceGenerator.DSM_OBJECT); List> medicalRecords = (List) ((Map) objectMap.get(SourceGenerator.DSM_OBJECT)).get("medicalRecords"); Optional> first = medicalRecords.stream() - .filter(i -> i.get(TestPatchUtil.MEDICAL_RECORD_COLUMN) != null) + .filter(i -> i.get(Util.underscoresToCamelCase(TestPatchUtil.MEDICAL_RECORD_COLUMN)) != null) .findFirst(); - first.ifPresentOrElse(val -> Assert.assertEquals(1L, val.get(TestPatchUtil.MEDICAL_RECORD_COLUMN)), Assert::fail); + first.ifPresentOrElse(val -> Assert.assertEquals(1L, val.get(Util.underscoresToCamelCase(TestPatchUtil.MEDICAL_RECORD_COLUMN))), Assert::fail); } @Test @@ -54,12 +55,12 @@ public void generateFromJson() { .get(SourceGenerator.DSM_OBJECT)) .get("tissueRecords"); Optional> maybeDdpInstance = tissueRecords.stream() - .filter(i -> i.get("DDP_INSTANCE") != null) + .filter(i -> i.get(Util.underscoresToCamelCase("DDP_INSTANCE")) != null) .findFirst(); - maybeDdpInstance.ifPresentOrElse(m -> Assert.assertEquals("TEST", m.get("DDP_INSTANCE")), Assert::fail); + maybeDdpInstance.ifPresentOrElse(m -> Assert.assertEquals("TEST", m.get(Util.underscoresToCamelCase("DDP_INSTANCE"))), Assert::fail); } - private static class TestSourceGenerator extends SourceGenerator { + private static class TestSourceGenerator extends CollectionSourceGenerator { public TestSourceGenerator(Parser parser, GeneratorPayload generatorPayload) { super(parser,generatorPayload); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorTest.java index 805e96e35..8047e05cd 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorTest.java @@ -99,7 +99,8 @@ public void updateIfExists() throws IOException { } private Collector instance(GeneratorPayload generatorPayload, NameValue nameValue) { - SourceGenerator sourceGenerator = new SourceGenerator(new ValueParser(), generatorPayload); +// SourceGenerator sourceGenerator = new SourceGenerator(new ValueParser(), generatorPayload); + SourceGenerator sourceGenerator = null; sourceGenerator.setDBElement(TestPatchUtil.getColumnNameMap().get(nameValue.getName())); return sourceGenerator; } From abb142cf8619521b04cd779db083071a6e11e1bc Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 23 Nov 2021 11:08:50 +0400 Subject: [PATCH 145/405] DDP-7121 | continue building factory for mapping exporters --- .../elastic/export/ElasticMappingExportAdapter.java | 5 ++++- .../dsm/model/elastic/export/ExportFacade.java | 6 ++++-- .../dsm/model/elastic/export/MappingExporterFactory.java | 9 ++++----- .../model/elastic/migration/ParticipantDataMigrator.java | 5 +++-- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticMappingExportAdapter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticMappingExportAdapter.java index e47cf7001..c2038ae2a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticMappingExportAdapter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticMappingExportAdapter.java @@ -15,8 +15,11 @@ public class ElasticMappingExportAdapter extends BaseExporter { private UpsertMappingRequestPayload upsertMappingRequestPayload; private Map mapping; - public ElasticMappingExportAdapter(UpsertMappingRequestPayload upsertMappingRequestPayload, Map mapping) { + public void setUpsertMappingRequestPayload(UpsertMappingRequestPayload upsertMappingRequestPayload) { this.upsertMappingRequestPayload = upsertMappingRequestPayload; + } + + public void setMapping(Map mapping) { this.mapping = mapping; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 52f15db3a..0198a42f6 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -26,7 +26,7 @@ public class ExportFacade { private static final Logger logger = LoggerFactory.getLogger(ExportFacade.class); - BaseExporter exportable; + Exportable exportable; Generator generator; ElasticSearchable searchable; private ExportFacadePayload exportFacadePayload; @@ -45,11 +45,13 @@ public void export() { private void upsertMapping() { // generator = new MappingGenerator(new TypeParser(), exportFacadePayload.getGeneratorPayload()); generator = null; + // mapping Map mappingToUpsert = generator.generate(); UpsertMappingRequestPayload upsertMappingRequestPayload = new UpsertMappingRequestPayload(exportFacadePayload.getIndex()); BaseGenerator.PropertyInfo propertyInfo = getPropertyInfo(); propertyInfo.setFieldName(Util.underscoresToCamelCase(exportFacadePayload.getFieldName())); - exportable = new ElasticMappingExportAdapter(upsertMappingRequestPayload, mappingToUpsert); + ExportableFactory mappingExporterFactory = new MappingExporterFactory(); + exportable = mappingExporterFactory.make(propertyInfo); exportable.export(); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingExporterFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingExporterFactory.java index 595c2e5e7..edc4ff457 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingExporterFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingExporterFactory.java @@ -3,17 +3,16 @@ import java.util.Objects; import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; +import org.broadinstitute.dsm.model.elastic.export.generate.CollectionMappingGenerator; +import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; public class MappingExporterFactory implements ExportableFactory { - @Override public Exportable make(BaseGenerator.PropertyInfo propertyInfo) { BaseExporter exporter = new NullObjectExporter(); - if (Objects.requireNonNull(propertyInfo).isCollection()) { - if (propertyInfo.getFieldName()) - } else { - + if (!propertyInfo.getFieldName().equals("followUps")) { + exporter = new ElasticMappingExportAdapter(); } return exporter; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java index 3a953194e..e925a91ba 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java @@ -59,8 +59,9 @@ private void exportMapping() { } catch (NoSuchFieldException | IllegalAccessException e) { throw new RuntimeException(e); } - ElasticMappingExportAdapter mappingExporter = new ElasticMappingExportAdapter(new UpsertMappingRequestPayload( - index), mapping); + ElasticMappingExportAdapter mappingExporter = new ElasticMappingExportAdapter(); + mappingExporter.setUpsertMappingRequestPayload(new UpsertMappingRequestPayload(index)); + mappingExporter.setMapping(mapping); mappingExporter.export(); } } From 957b3cd922293fd26f26ab6eacbc9ed5a777529d Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 23 Nov 2021 11:38:33 +0400 Subject: [PATCH 146/405] DDP-7121 | add abstract methos to exportable to provide more flexibility, make general class RequestPayload instead of separate classes for mapping and data exporter --- .../model/elastic/export/BaseExporter.java | 15 ++++ .../export/ElasticDataExportAdapter.java | 14 +--- .../export/ElasticMappingExportAdapter.java | 15 +--- .../model/elastic/export/ExportFacade.java | 17 ++-- .../dsm/model/elastic/export/Exportable.java | 7 ++ .../export/MappingExporterFactory.java | 4 - .../model/elastic/export/RequestPayload.java | 24 ++++++ .../export/UpsertDataRequestPayload.java | 78 ------------------- .../export/UpsertMappingRequestPayload.java | 17 ---- .../MedicalRecordMappingGenerator.java | 21 ----- .../migration/ParticipantDataMigrator.java | 4 +- .../export/generate/MappingGeneratorTest.java | 10 --- 12 files changed, 61 insertions(+), 165 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/RequestPayload.java delete mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/UpsertDataRequestPayload.java delete mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/UpsertMappingRequestPayload.java delete mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MedicalRecordMappingGenerator.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java index a2b4f65d9..912e45253 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java @@ -3,8 +3,23 @@ import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.elasticsearch.client.RestHighLevelClient; +import java.util.Map; import java.util.Objects; public abstract class BaseExporter implements Exportable { protected RestHighLevelClient clientInstance = ElasticSearchUtil.getClientInstance(); + + protected RequestPayload requestPayload; + protected Map source; + + @Override + public void setRequestPayload(RequestPayload requestPayload) { + this.requestPayload = requestPayload; + } + + @Override + public void setSource(Map source) { + this.source = source; + } + } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticDataExportAdapter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticDataExportAdapter.java index 5cf7b2c78..115904b89 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticDataExportAdapter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticDataExportAdapter.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.util.Map; +import org.broadinstitute.dsm.model.elastic.Util; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.RequestOptions; import org.slf4j.Logger; @@ -12,19 +13,12 @@ public class ElasticDataExportAdapter extends BaseExporter { private static final Logger logger = LoggerFactory.getLogger(ElasticDataExportAdapter.class); - private UpsertDataRequestPayload upsertDataRequestPayload; - private Map data; - - public ElasticDataExportAdapter(UpsertDataRequestPayload upsertDataRequestPayload, Map data) { - this.upsertDataRequestPayload = upsertDataRequestPayload; - this.data = data; - } - - @Override public void export() { logger.info("initialize exporting data to ES"); - UpdateRequest updateRequest = upsertDataRequestPayload.getUpdateRequest(data); + UpdateRequest updateRequest = new UpdateRequest(requestPayload.getIndex(), Util.DOC, requestPayload.getId()) + .retryOnConflict(5) + .docAsUpsert(true); try { clientInstance.update(updateRequest, RequestOptions.DEFAULT); } catch (IOException e) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticMappingExportAdapter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticMappingExportAdapter.java index c2038ae2a..a7712d94b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticMappingExportAdapter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticMappingExportAdapter.java @@ -12,22 +12,11 @@ public class ElasticMappingExportAdapter extends BaseExporter { private static final Logger logger = LoggerFactory.getLogger(ElasticMappingExportAdapter.class); - private UpsertMappingRequestPayload upsertMappingRequestPayload; - private Map mapping; - - public void setUpsertMappingRequestPayload(UpsertMappingRequestPayload upsertMappingRequestPayload) { - this.upsertMappingRequestPayload = upsertMappingRequestPayload; - } - - public void setMapping(Map mapping) { - this.mapping = mapping; - } - @Override public void export() { logger.info("initialize exporting mapping to ES"); - PutMappingRequest putMappingRequest = upsertMappingRequestPayload.getPutMappingRequest(); - putMappingRequest.source(mapping); + PutMappingRequest putMappingRequest = new PutMappingRequest(requestPayload.getIndex()); + putMappingRequest.source(source); try { clientInstance.indices().putMapping(putMappingRequest, RequestOptions.DEFAULT); } catch (IOException e) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 0198a42f6..96e660be7 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -43,15 +43,14 @@ public void export() { } private void upsertMapping() { -// generator = new MappingGenerator(new TypeParser(), exportFacadePayload.getGeneratorPayload()); - generator = null; - // mapping Map mappingToUpsert = generator.generate(); - UpsertMappingRequestPayload upsertMappingRequestPayload = new UpsertMappingRequestPayload(exportFacadePayload.getIndex()); + RequestPayload upsertMappingRequestPayload = new RequestPayload(exportFacadePayload.getIndex()); BaseGenerator.PropertyInfo propertyInfo = getPropertyInfo(); propertyInfo.setFieldName(Util.underscoresToCamelCase(exportFacadePayload.getFieldName())); ExportableFactory mappingExporterFactory = new MappingExporterFactory(); exportable = mappingExporterFactory.make(propertyInfo); + exportable.setRequestPayload(upsertMappingRequestPayload); + exportable.setSource(mappingToUpsert); exportable.export(); } @@ -88,13 +87,11 @@ private BaseGenerator.PropertyInfo getPropertyInfo() { } private void upsertData(Map elasticDataToExport) { - UpsertDataRequestPayload upsertDataRequestPayload = new UpsertDataRequestPayload.Builder(exportFacadePayload.getIndex(), - exportFacadePayload.getDocId()) - .withDocAsUpsert(true) - .withRetryOnConflict(5) - .build(); + RequestPayload requestPayload = new RequestPayload(exportFacadePayload.getIndex(), exportFacadePayload.getDocId()); logger.info("Built upsert data request payload"); - exportable = new ElasticDataExportAdapter(upsertDataRequestPayload, elasticDataToExport); + exportable = new ElasticDataExportAdapter(); + exportable.setRequestPayload(requestPayload); + exportable.setSource(elasticDataToExport); exportable.export(); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java index f0e6a6d28..eb075a00f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java @@ -1,12 +1,19 @@ package org.broadinstitute.dsm.model.elastic.export; +import java.util.Map; + import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.util.ParticipantUtil; public interface Exportable { + void export(); + void setSource(Map source); + + void setRequestPayload(RequestPayload requestPayload); + default String getParticipantGuid(String participantId, String index) { if (!(ParticipantUtil.isGuid(participantId))) { ElasticSearchParticipantDto participantById = diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingExporterFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingExporterFactory.java index edc4ff457..3dee6d2cf 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingExporterFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingExporterFactory.java @@ -1,10 +1,6 @@ package org.broadinstitute.dsm.model.elastic.export; -import java.util.Objects; - import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; -import org.broadinstitute.dsm.model.elastic.export.generate.CollectionMappingGenerator; -import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; public class MappingExporterFactory implements ExportableFactory { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/RequestPayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/RequestPayload.java new file mode 100644 index 000000000..976310686 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/RequestPayload.java @@ -0,0 +1,24 @@ +package org.broadinstitute.dsm.model.elastic.export; + +public class RequestPayload { + + private String index; + private String id; + + public RequestPayload(String index, String id) { + this.index = index; + this.id = id; + } + + public RequestPayload(String index) { + this(index, ""); + } + + public String getIndex() { + return index; + } + + public String getId() { + return id; + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/UpsertDataRequestPayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/UpsertDataRequestPayload.java deleted file mode 100644 index b7102f020..000000000 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/UpsertDataRequestPayload.java +++ /dev/null @@ -1,78 +0,0 @@ -package org.broadinstitute.dsm.model.elastic.export; - -import org.broadinstitute.dsm.model.elastic.Util; -import org.elasticsearch.action.update.UpdateRequest; - -import java.util.Map; - -public class UpsertDataRequestPayload { - - private String index; - private String type; - private String id; - private boolean docAsUpsert; - private int retryOnConflict; - - private UpsertDataRequestPayload(Builder builder) { - this.index = builder.index; - this.type = builder.type; - this.id = builder.id; - this.docAsUpsert = builder.docAsUpsert; - this.retryOnConflict = builder.retryOnConflict; - } - - public UpdateRequest getUpdateRequest(Map data) { - return new UpdateRequest() - .index(index) - .type(Util.DOC) - .id(id) - .doc(data) - .docAsUpsert(docAsUpsert) - .retryOnConflict(retryOnConflict); - } - - public static class Builder { - private String index; - private String type; - private String id; - private boolean docAsUpsert = true; - private int retryOnConflict; - - public Builder(String index, String id) { - this.index = index; - this.id = id; - } - - public Builder withIndex(String index) { - this.index = index; - return this; - } - - public Builder withType(String type) { - this.type = type; - return this; - } - - public Builder withId(String id) { - this.id = id; - return this; - } - - public Builder withDocAsUpsert(boolean docAsUpsert) { - this.docAsUpsert = docAsUpsert; - return this; - } - - public Builder withRetryOnConflict(int retryOnConflict) { - this.retryOnConflict = retryOnConflict; - return this; - } - - public UpsertDataRequestPayload build() { - return new UpsertDataRequestPayload(this); - } - - - } - -} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/UpsertMappingRequestPayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/UpsertMappingRequestPayload.java deleted file mode 100644 index 8134f4e48..000000000 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/UpsertMappingRequestPayload.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.broadinstitute.dsm.model.elastic.export; - -import org.elasticsearch.client.indices.PutMappingRequest; - -public class UpsertMappingRequestPayload { - - private String index; - - public UpsertMappingRequestPayload(String index) { - this.index = index; - } - - public PutMappingRequest getPutMappingRequest() { - return new PutMappingRequest(index); - } - -} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MedicalRecordMappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MedicalRecordMappingGenerator.java deleted file mode 100644 index 1f89f0b0f..000000000 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MedicalRecordMappingGenerator.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.broadinstitute.dsm.model.elastic.export.generate; - -import java.util.Map; - -import org.broadinstitute.dsm.model.elastic.export.parse.Parser; - -public class MedicalRecordMappingGenerator extends CollectionMappingGenerator{ - public MedicalRecordMappingGenerator(Parser parser, - GeneratorPayload generatorPayload) { - super(parser, generatorPayload); - } - - @Override - public Object collect() { - Map collect = (Map) super.collect(); - if ("followUps".equals(generatorPayload.getName())) { - collect.remove("followUps"); - } - return collect; - } -} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java index e925a91ba..0385f3928 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java @@ -10,7 +10,7 @@ import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.elastic.export.ElasticMappingExportAdapter; -import org.broadinstitute.dsm.model.elastic.export.UpsertMappingRequestPayload; +import org.broadinstitute.dsm.model.elastic.export.RequestPayload; import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; import org.broadinstitute.dsm.model.elastic.export.generate.MappingGenerator; import org.broadinstitute.dsm.model.elastic.export.parse.BaseParser; @@ -60,7 +60,7 @@ private void exportMapping() { throw new RuntimeException(e); } ElasticMappingExportAdapter mappingExporter = new ElasticMappingExportAdapter(); - mappingExporter.setUpsertMappingRequestPayload(new UpsertMappingRequestPayload(index)); + mappingExporter.setUpsertMappingRequestPayload(new RequestPayload(index)); mappingExporter.setMapping(mapping); mappingExporter.export(); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java index 7a6d94bfd..4ff009c46 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java @@ -116,16 +116,6 @@ public void merge() { Assert.assertFalse(Objects.isNull(value)); } - @Test - public void removeFollowUpsFromConstruct() { - NameValue nameValue = new NameValue("m.followUps", "TEST"); - GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, 0); - TypeParser parser = new TypeParser(); - MappingGenerator mappingGenerator = new MedicalRecordMappingGenerator(parser, generatorPayload); - Map objectMap = mappingGenerator.generate(); - - } - private String extractDeepestLeveleValue(Map objectMap) { return (String) ((Map) From 0d9383c964a973925fb42f8d708aa7b276185f1a Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 23 Nov 2021 16:31:30 +0400 Subject: [PATCH 147/405] DDP-7121 | refactor based on new abstractions --- .../dsm/model/elastic/migration/BaseMigrator.java | 3 ++- .../dsm/model/elastic/migration/ParticipantDataMigrator.java | 4 ++-- .../model/elastic/export/generate/MappingGeneratorTest.java | 1 + .../model/elastic/export/process/CollectionProcessorTest.java | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseMigrator.java index 5548cfc96..44c28ccbf 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseMigrator.java @@ -2,13 +2,14 @@ import java.util.Map; +import org.broadinstitute.dsm.model.elastic.export.BaseExporter; import org.broadinstitute.dsm.model.elastic.export.Exportable; import org.broadinstitute.dsm.model.elastic.export.generate.Generator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import spark.utils.StringUtils; -public abstract class BaseMigrator implements Exportable, Generator { +public abstract class BaseMigrator extends BaseExporter implements Generator { private static final Logger logger = LoggerFactory.getLogger(BaseMigrator.class); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java index 0385f3928..1cad171bf 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java @@ -60,8 +60,8 @@ private void exportMapping() { throw new RuntimeException(e); } ElasticMappingExportAdapter mappingExporter = new ElasticMappingExportAdapter(); - mappingExporter.setUpsertMappingRequestPayload(new RequestPayload(index)); - mappingExporter.setMapping(mapping); + mappingExporter.setRequestPayload(new RequestPayload(index)); + mappingExporter.setSource(mapping); mappingExporter.export(); } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java index 4ff009c46..0c090d12f 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java @@ -169,6 +169,7 @@ public static TestSingleMappingGenerator of(GeneratorPayload generatorPayload) { } + private static class TestCollectionMappingGenerator extends CollectionMappingGenerator { public TestCollectionMappingGenerator(Parser typeParser, GeneratorPayload generatorPayload) { diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorTest.java index 8047e05cd..3f5088673 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorTest.java @@ -109,7 +109,7 @@ private static class TestCollectionProcessor extends CollectionProcessor { public TestCollectionProcessor(ESDsm esDsm, String propertyName, GeneratorPayload generatorPayload, Collector collector) { - super(esDsm, propertyName, generatorPayload, collector); + super(esDsm, propertyName, generatorPayload.getRecordId(), collector); } } From 3bea069624e15bc121c7df4b92917150887e579e Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 23 Nov 2021 17:53:47 +0400 Subject: [PATCH 148/405] DDP-7121 | add factory method abstraction for source and mapping generator --- .../model/elastic/export/ExportFacade.java | 11 +++++++--- .../export/generate/BaseGenerator.java | 16 +++++++++++++- .../generate/CollectionMappingGenerator.java | 2 ++ .../elastic/export/generate/Generator.java | 4 ++++ .../export/generate/GeneratorFactory.java | 7 ++++++ .../export/generate/MappingGenerator.java | 2 ++ .../generate/MappingGeneratorFactory.java | 14 ++++++++++++ .../export/generate/NullObjectGenerator.java | 22 +++++++++++++++++++ .../generate/SingleMappingGenerator.java | 2 ++ 9 files changed, 76 insertions(+), 4 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorFactory.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorFactory.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/NullObjectGenerator.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 96e660be7..4328c492c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -9,7 +9,9 @@ import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; import org.broadinstitute.dsm.model.elastic.export.generate.Generator; +import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorFactory; import org.broadinstitute.dsm.model.elastic.export.generate.MappingGenerator; +import org.broadinstitute.dsm.model.elastic.export.generate.MappingGeneratorFactory; import org.broadinstitute.dsm.model.elastic.export.generate.SourceGenerator; import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; import org.broadinstitute.dsm.model.elastic.export.parse.ValueParser; @@ -43,9 +45,11 @@ public void export() { } private void upsertMapping() { + BaseGenerator.PropertyInfo propertyInfo = getPropertyInfo(); + GeneratorFactory generatorFactory = new MappingGeneratorFactory(); + generator = generatorFactory.make(propertyInfo); Map mappingToUpsert = generator.generate(); RequestPayload upsertMappingRequestPayload = new RequestPayload(exportFacadePayload.getIndex()); - BaseGenerator.PropertyInfo propertyInfo = getPropertyInfo(); propertyInfo.setFieldName(Util.underscoresToCamelCase(exportFacadePayload.getFieldName())); ExportableFactory mappingExporterFactory = new MappingExporterFactory(); exportable = mappingExporterFactory.make(propertyInfo); @@ -64,8 +68,9 @@ private ESDsm fetchData() { private Map processData(ESDsm esDsm) { BaseGenerator.PropertyInfo propertyInfo = getPropertyInfo(); ValueParser valueParser = new ValueParser(); -// SourceGenerator sourceGenerator = new SourceGenerator(valueParser, exportFacadePayload.getGeneratorPayload()); - SourceGenerator sourceGenerator = null; + SourceGenerator sourceGenerator = new SourceGenerator(valueParser, exportFacadePayload.getGeneratorPayload()); + GeneratorFactory generatorFactory = new MappingGeneratorFactory(); + generator = generatorFactory.make(propertyInfo); this.generator = sourceGenerator; Map dataToReturn = generator.generate(); logger.info("Processing ES participant data"); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java index 2efc66e24..8cb86e784 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java @@ -21,7 +21,7 @@ public abstract class BaseGenerator implements Generator, Collector { public static final String DSM_OBJECT = "dsm"; public static final String PROPERTIES = "properties"; protected static final Gson GSON = new Gson(); - protected final Parser parser; + protected Parser parser; protected GeneratorPayload generatorPayload; private DBElement dbElement; @@ -31,6 +31,20 @@ public BaseGenerator(Parser parser, GeneratorPayload generatorPayload) { dbElement = Util.getDBElement(getNameValue().getName()); } + public BaseGenerator() { + + } + + @Override + public void setParser(Parser parser) { + this.parser = Objects.requireNonNull(parser); + } + + @Override + public void setPayload(GeneratorPayload generatorPayload) { + this.generatorPayload = Objects.requireNonNull(generatorPayload); + } + //setter method to set dbElement for testing only!!! public void setDBElement(DBElement dbElement) { this.dbElement = dbElement; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java index b8e497149..8b086ea8e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java @@ -12,6 +12,8 @@ public CollectionMappingGenerator(Parser parser, GeneratorPayload generatorPaylo super(parser, generatorPayload); } + public CollectionMappingGenerator() {} + @Override protected Map getElement(Object type) { return Map.of( diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Generator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Generator.java index 01cc63886..319ae34b5 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Generator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Generator.java @@ -2,6 +2,10 @@ import java.util.Map; +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; + public interface Generator { Map generate(); + void setParser(Parser parser); + void setPayload(GeneratorPayload generatorPayload); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorFactory.java new file mode 100644 index 000000000..69d9d1eea --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorFactory.java @@ -0,0 +1,7 @@ +package org.broadinstitute.dsm.model.elastic.export.generate; + +public interface GeneratorFactory { + + Generator make(BaseGenerator.PropertyInfo propertyInfo); + +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java index 24ccc172b..52519ea78 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java @@ -20,6 +20,8 @@ public MappingGenerator(Parser parser, GeneratorPayload generatorPayload) { super(parser, generatorPayload); } + public MappingGenerator() {} + @Override public Map generate() { logger.info("preparing mapping to upsert"); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorFactory.java new file mode 100644 index 000000000..dcd79ca3e --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorFactory.java @@ -0,0 +1,14 @@ +package org.broadinstitute.dsm.model.elastic.export.generate; + +public class MappingGeneratorFactory implements GeneratorFactory { + @Override + public Generator make(BaseGenerator.PropertyInfo propertyInfo) { + Generator generator = new NullObjectGenerator(); + if (propertyInfo.isCollection()) { + generator = new CollectionMappingGenerator(); + } else { + generator = new SingleMappingGenerator(); + } + return generator; + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/NullObjectGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/NullObjectGenerator.java new file mode 100644 index 000000000..b639d2a3b --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/NullObjectGenerator.java @@ -0,0 +1,22 @@ +package org.broadinstitute.dsm.model.elastic.export.generate; + +import java.util.Map; + +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; + +public class NullObjectGenerator implements Generator { + @Override + public Map generate() { + return Map.of(); + } + + @Override + public void setParser(Parser parser) { + + } + + @Override + public void setPayload(GeneratorPayload generatorPayload) { + + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java index 5f290afe3..9f73aab5e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java @@ -12,6 +12,8 @@ public SingleMappingGenerator(Parser parser, GeneratorPayload generatorPayload) super(parser, generatorPayload); } + public SingleMappingGenerator() {} + @Override protected Map getElement(Object type) { return Map.of(Util.underscoresToCamelCase(getDBElement().getColumnName()), type); From 18935c5dac164ec97a7a4c28917c2166a915aa6e Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 23 Nov 2021 18:53:36 +0400 Subject: [PATCH 149/405] DDP-7121 | create SourceGeneratorFactory and make further modifications based on new classes --- .../export/ElasticDataExportAdapter.java | 4 +-- .../model/elastic/export/ExportFacade.java | 25 ++++++++----------- .../elastic/export/ExportFacadePayload.java | 11 +++++++- .../export/generate/BaseGenerator.java | 2 ++ .../generate/CollectionSourceGenerator.java | 4 +++ .../export/generate/GeneratorFactory.java | 2 +- .../generate/MappingGeneratorFactory.java | 4 +-- .../generate/SingleSourceGenerator.java | 4 +++ .../export/generate/SourceGenerator.java | 3 +++ .../generate/SourceGeneratorFactory.java | 15 +++++++++++ .../migration/KitRequestShippingMigrator.java | 11 ++++++++ .../migration/MedicalRecordMigrator.java | 11 ++++++++ .../migration/OncHistoryDetailsMigrator.java | 11 ++++++++ .../elastic/migration/OncHistoryMigrator.java | 12 +++++++++ .../migration/ParticipantDataMigrator.java | 11 ++++++++ .../migration/ParticipantMigrator.java | 12 +++++++++ .../elastic/migration/TissueMigrator.java | 12 +++++++++ .../migration/BaseCollectionMigratorTest.java | 12 +++++++++ 18 files changed, 146 insertions(+), 20 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorFactory.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticDataExportAdapter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticDataExportAdapter.java index 115904b89..a8364de58 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticDataExportAdapter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticDataExportAdapter.java @@ -17,8 +17,8 @@ public class ElasticDataExportAdapter extends BaseExporter { public void export() { logger.info("initialize exporting data to ES"); UpdateRequest updateRequest = new UpdateRequest(requestPayload.getIndex(), Util.DOC, requestPayload.getId()) - .retryOnConflict(5) - .docAsUpsert(true); + .doc(source) + .retryOnConflict(5); try { clientInstance.update(updateRequest, RequestOptions.DEFAULT); } catch (IOException e) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 4328c492c..f21b460d2 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -7,12 +7,7 @@ import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.elastic.ESDsm; import org.broadinstitute.dsm.model.elastic.Util; -import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; -import org.broadinstitute.dsm.model.elastic.export.generate.Generator; -import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorFactory; -import org.broadinstitute.dsm.model.elastic.export.generate.MappingGenerator; -import org.broadinstitute.dsm.model.elastic.export.generate.MappingGeneratorFactory; -import org.broadinstitute.dsm.model.elastic.export.generate.SourceGenerator; +import org.broadinstitute.dsm.model.elastic.export.generate.*; import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; import org.broadinstitute.dsm.model.elastic.export.parse.ValueParser; import org.broadinstitute.dsm.model.elastic.export.process.CollectionProcessor; @@ -29,7 +24,7 @@ public class ExportFacade { private static final Logger logger = LoggerFactory.getLogger(ExportFacade.class); Exportable exportable; - Generator generator; + BaseGenerator generator; ElasticSearchable searchable; private ExportFacadePayload exportFacadePayload; Processor processor; @@ -48,9 +43,11 @@ private void upsertMapping() { BaseGenerator.PropertyInfo propertyInfo = getPropertyInfo(); GeneratorFactory generatorFactory = new MappingGeneratorFactory(); generator = generatorFactory.make(propertyInfo); + generator.setParser(new TypeParser()); + generator.setPayload(exportFacadePayload.getGeneratorPayload()); Map mappingToUpsert = generator.generate(); RequestPayload upsertMappingRequestPayload = new RequestPayload(exportFacadePayload.getIndex()); - propertyInfo.setFieldName(Util.underscoresToCamelCase(exportFacadePayload.getFieldName())); + propertyInfo.setFieldName(Util.underscoresToCamelCase(exportFacadePayload.getColumnName())); ExportableFactory mappingExporterFactory = new MappingExporterFactory(); exportable = mappingExporterFactory.make(propertyInfo); exportable.setRequestPayload(upsertMappingRequestPayload); @@ -68,15 +65,15 @@ private ESDsm fetchData() { private Map processData(ESDsm esDsm) { BaseGenerator.PropertyInfo propertyInfo = getPropertyInfo(); ValueParser valueParser = new ValueParser(); - SourceGenerator sourceGenerator = new SourceGenerator(valueParser, exportFacadePayload.getGeneratorPayload()); - GeneratorFactory generatorFactory = new MappingGeneratorFactory(); - generator = generatorFactory.make(propertyInfo); - this.generator = sourceGenerator; + GeneratorFactory sourceGeneratorFactory = new SourceGeneratorFactory(); + generator = sourceGeneratorFactory.make(propertyInfo); + generator.setParser(valueParser); + generator.setPayload(exportFacadePayload.getGeneratorPayload()); Map dataToReturn = generator.generate(); logger.info("Processing ES participant data"); if (propertyInfo.isCollection()) { processor = new CollectionProcessor(esDsm, propertyInfo.getPropertyName(), exportFacadePayload.getRecordId(), - sourceGenerator); + generator); List> processedData = processor.process(); if (!processedData.isEmpty()) { dataToReturn = Map.of(MappingGenerator.DSM_OBJECT, Map.of(propertyInfo.getPropertyName(), processedData)); @@ -87,7 +84,7 @@ private Map processData(ESDsm esDsm) { } private BaseGenerator.PropertyInfo getPropertyInfo() { - DBElement dbElement = PatchUtil.getColumnNameMap().get(exportFacadePayload.getFieldName()); + DBElement dbElement = PatchUtil.getColumnNameMap().get(exportFacadePayload.getFieldNameWithAlias()); return Util.TABLE_ALIAS_MAPPINGS.get(dbElement.getTableAlias()); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java index eba81b099..16a5bf180 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java @@ -8,6 +8,7 @@ import org.broadinstitute.dsm.util.PatchUtil; public class ExportFacadePayload { + private String index; private String docId; private GeneratorPayload generatorPayload; @@ -34,8 +35,16 @@ public void setDocId(String docId) { this.docId = docId; } - public String getFieldName() { + public String getColumnName() { DBElement dbElement = PatchUtil.getColumnNameMap().get(generatorPayload.getName()); return dbElement.getColumnName(); } + + public GeneratorPayload getGeneratorPayload() { + return generatorPayload; + } + + public String getFieldNameWithAlias() { + return generatorPayload.getName(); + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java index 8cb86e784..80edeebec 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java @@ -56,6 +56,8 @@ protected NameValue getNameValue() { //wrap Util.getDBElement in protected method so that we can override it in testing class for tests protected DBElement getDBElement() { + if (dbElement == null) + dbElement = Util.getDBElement(getNameValue().getName()); return dbElement; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java index 0eb937b85..cd9b71a6b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java @@ -18,6 +18,10 @@ public CollectionSourceGenerator(Parser parser, super(parser, generatorPayload); } + public CollectionSourceGenerator() { + + } + @Override protected Object construct() { logger.info("Constructing nested data"); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorFactory.java index 69d9d1eea..396359107 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorFactory.java @@ -2,6 +2,6 @@ public interface GeneratorFactory { - Generator make(BaseGenerator.PropertyInfo propertyInfo); + BaseGenerator make(BaseGenerator.PropertyInfo propertyInfo); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorFactory.java index dcd79ca3e..ede16cfc6 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorFactory.java @@ -2,8 +2,8 @@ public class MappingGeneratorFactory implements GeneratorFactory { @Override - public Generator make(BaseGenerator.PropertyInfo propertyInfo) { - Generator generator = new NullObjectGenerator(); + public BaseGenerator make(BaseGenerator.PropertyInfo propertyInfo) { + BaseGenerator generator; if (propertyInfo.isCollection()) { generator = new CollectionMappingGenerator(); } else { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleSourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleSourceGenerator.java index ecb1ac420..4cc4e8379 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleSourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleSourceGenerator.java @@ -16,6 +16,10 @@ public SingleSourceGenerator(Parser parser, super(parser, generatorPayload); } + public SingleSourceGenerator() { + + } + @Override protected Object construct() { return parseJsonValuesToObject(); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java index 166155ec5..7b7a5bfb3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java @@ -17,6 +17,9 @@ public SourceGenerator(Parser parser, GeneratorPayload generatorPayload) { super(parser, generatorPayload); } + public SourceGenerator() { + + } @Override public Map generate() { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorFactory.java new file mode 100644 index 000000000..c5e0866af --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorFactory.java @@ -0,0 +1,15 @@ +package org.broadinstitute.dsm.model.elastic.export.generate; + +public class SourceGeneratorFactory implements GeneratorFactory { + + @Override + public BaseGenerator make(BaseGenerator.PropertyInfo propertyInfo) { + BaseGenerator generator; + if (propertyInfo.isCollection()) { + generator = new CollectionSourceGenerator(); + } else { + generator = new SingleSourceGenerator(); + } + return generator; + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/KitRequestShippingMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/KitRequestShippingMigrator.java index ded3df947..8969f527f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/KitRequestShippingMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/KitRequestShippingMigrator.java @@ -4,6 +4,8 @@ import java.util.Map; import org.broadinstitute.dsm.db.KitRequestShipping; +import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import org.broadinstitute.dsm.statics.ESObjectConstants; public class KitRequestShippingMigrator extends BaseCollectionMigrator { @@ -18,4 +20,13 @@ protected Map getDataByRealm() { return (Map) kitRequests; } + @Override + public void setParser(Parser parser) { + + } + + @Override + public void setPayload(GeneratorPayload generatorPayload) { + + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigrator.java index c34197469..c135fab59 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigrator.java @@ -3,6 +3,8 @@ import java.util.Map; import org.broadinstitute.dsm.db.MedicalRecord; +import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import org.broadinstitute.dsm.statics.ESObjectConstants; public class MedicalRecordMigrator extends BaseCollectionMigrator { @@ -16,4 +18,13 @@ protected Map getDataByRealm() { return (Map) MedicalRecord.getMedicalRecords(realm); } + @Override + public void setParser(Parser parser) { + + } + + @Override + public void setPayload(GeneratorPayload generatorPayload) { + + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java index fd5a54290..abb5233b3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java @@ -3,6 +3,8 @@ import java.util.Map; import org.broadinstitute.dsm.db.OncHistoryDetail; +import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import org.broadinstitute.dsm.statics.ESObjectConstants; @@ -17,4 +19,13 @@ protected Map getDataByRealm() { return (Map) OncHistoryDetail.getOncHistoryDetails(realm); } + @Override + public void setParser(Parser parser) { + + } + + @Override + public void setPayload(GeneratorPayload generatorPayload) { + + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryMigrator.java index 943bc91af..48656c773 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryMigrator.java @@ -3,6 +3,8 @@ import java.util.Map; import org.broadinstitute.dsm.db.dao.ddp.onchistory.OncHistoryDao; +import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import org.broadinstitute.dsm.statics.ESObjectConstants; public class OncHistoryMigrator extends BaseSingleMigrator { @@ -15,4 +17,14 @@ public OncHistoryMigrator(String index, String realm) { protected Map getDataByRealm() { return (Map) new OncHistoryDao().getOncHistoriesByStudy(realm); } + + @Override + public void setParser(Parser parser) { + + } + + @Override + public void setPayload(GeneratorPayload generatorPayload) { + + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java index 1cad171bf..bc7e63890 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java @@ -14,6 +14,7 @@ import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; import org.broadinstitute.dsm.model.elastic.export.generate.MappingGenerator; import org.broadinstitute.dsm.model.elastic.export.parse.BaseParser; +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; import org.broadinstitute.dsm.statics.ESObjectConstants; @@ -64,4 +65,14 @@ private void exportMapping() { mappingExporter.setSource(mapping); mappingExporter.export(); } + + @Override + public void setParser(Parser parser) { + + } + + @Override + public void setPayload(GeneratorPayload generatorPayload) { + + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantMigrator.java index 3adabc6fe..6d718171d 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantMigrator.java @@ -6,6 +6,8 @@ import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.Exportable; import org.broadinstitute.dsm.model.elastic.export.generate.Generator; +import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import org.broadinstitute.dsm.statics.ESObjectConstants; public class ParticipantMigrator extends BaseSingleMigrator implements Exportable, Generator { @@ -18,4 +20,14 @@ public ParticipantMigrator(String index, String realm) { protected Map getDataByRealm() { return (Map) Participant.getParticipants(realm); } + + @Override + public void setParser(Parser parser) { + + } + + @Override + public void setPayload(GeneratorPayload generatorPayload) { + + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigrator.java index 96beb9ca4..2a9995803 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigrator.java @@ -3,6 +3,8 @@ import java.util.Map; import org.broadinstitute.dsm.db.dao.ddp.tissue.TissueDao; +import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; public class TissueMigrator extends BaseCollectionMigrator { @@ -14,4 +16,14 @@ public TissueMigrator(String index, String realm) { protected Map getDataByRealm() { return (Map) new TissueDao().getTissuesByStudy(realm); } + + @Override + public void setParser(Parser parser) { + + } + + @Override + public void setPayload(GeneratorPayload generatorPayload) { + + } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigratorTest.java index 3b44b6b7b..196934bf6 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigratorTest.java @@ -7,6 +7,8 @@ import org.broadinstitute.dsm.db.OncHistoryDetail; import org.broadinstitute.dsm.db.Tissue; +import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import org.junit.Assert; import org.junit.Test; @@ -49,5 +51,15 @@ public MockBaseCollectionMigrator(String index, String realm, String object) { protected Map getDataByRealm() { return Map.of(); } + + @Override + public void setParser(Parser parser) { + + } + + @Override + public void setPayload(GeneratorPayload generatorPayload) { + + } } } \ No newline at end of file From fc14e6245bcdc71cff7f47746ff0624f36c713cf Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 24 Nov 2021 11:29:22 +0400 Subject: [PATCH 150/405] DDP-7121 | renaming to match elasticsearch fields --- .../org/broadinstitute/dsm/db/MedicalRecord.java | 10 ++++++---- .../dsm/export/WorkflowAndFamilyIdExporter.java | 6 +++--- .../broadinstitute/dsm/model/elastic/ESProfile.java | 6 +++--- .../org/broadinstitute/dsm/model/elastic/Util.java | 5 +++-- .../elastic/search/ElasticSearchParticipantDto.java | 8 +++++--- .../dsm/model/participant/ParticipantWrapper.java | 2 +- .../dsm/model/patch/ExistingRecordPatch.java | 9 ++++----- .../dsm/model/patch/ParticipantDataPatch.java | 3 +-- .../dsm/model/rgp/AutomaticProbandDataCreator.java | 12 ++++++------ .../org/broadinstitute/dsm/route/DashboardRoute.java | 2 +- .../org/broadinstitute/dsm/route/KitUploadRoute.java | 8 ++++---- .../broadinstitute/dsm/util/ElasticSearchUtil.java | 8 ++++---- .../org/broadinstitute/dsm/ElasticSearchTest.java | 4 ++-- .../dsm/model/elastic/search/ElasticSearchTest.java | 8 ++++---- .../model/participant/ParticipantWrapperTest.java | 4 ++-- .../broadinstitute/dsm/route/KitUploadRouteTest.java | 2 +- 16 files changed, 50 insertions(+), 47 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java index 3b890a325..6750a99fa 100644 --- a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java +++ b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java @@ -52,7 +52,7 @@ public class MedicalRecord { primaryKey = DBConstants.MEDICAL_RECORD_ID, columnPrefix = "") @ColumnName (DBConstants.MEDICAL_RECORD_ID) - private final String medicalRecordId; + private String medicalRecordId; @TableName ( name = DBConstants.DDP_INSTITUTION, @@ -307,7 +307,7 @@ public class MedicalRecord { primaryKey = DBConstants.MEDICAL_RECORD_ID, columnPrefix = "") @ColumnName (DBConstants.FOLLOW_UP_REQUESTS) - private List followUps; + private FollowUp[] followUps; @TableName ( name = DBConstants.DDP_MEDICAL_RECORD, @@ -334,6 +334,8 @@ public MedicalRecord(String medicalRecordId, String institutionId, String ddpIns this.type = type; } + public MedicalRecord() {} + public MedicalRecord(String medicalRecordId, String institutionId, String ddpInstitutionId, String type, String name, String contact, String phone, String fax, String faxSent, String faxSentBy, String faxConfirmed, @@ -342,7 +344,7 @@ public MedicalRecord(String medicalRecordId, String institutionId, String ddpIns String mrReceived, String mrDocument, String mrDocumentFileNames, boolean mrProblem, String mrProblemText, boolean unableObtain, boolean duplicate, boolean international, boolean crRequired, String pathologyPresent, String mrNotes, boolean reviewMedicalRecord, - List followUps, boolean followUpRequired, String followUpRequiredText, String additionalValues, + FollowUp[] followUps, boolean followUpRequired, String followUpRequiredText, String additionalValues, String mrUnableToObtainText, String ddpParticipantId) { this.medicalRecordId = medicalRecordId; this.institutionId = institutionId; @@ -412,7 +414,7 @@ public static MedicalRecord getMedicalRecord(@NonNull ResultSet rs) throws SQLEx rs.getString(DBConstants.PATHOLOGY_PRESENT), rs.getString(DBConstants.NOTES), rs.getBoolean(DBConstants.REVIEW_MEDICAL_RECORD), - new Gson().fromJson(rs.getString(DBConstants.FOLLOW_UP_REQUESTS), new TypeToken>(){}.getType()), + new Gson().fromJson(rs.getString(DBConstants.FOLLOW_UP_REQUESTS), FollowUp[].class), rs.getBoolean(DBConstants.FOLLOWUP_REQUIRED), rs.getString(DBConstants.FOLLOWUP_REQUIRED_TEXT), rs.getString(DBConstants.ADDITIONAL_VALUES), diff --git a/src/main/java/org/broadinstitute/dsm/export/WorkflowAndFamilyIdExporter.java b/src/main/java/org/broadinstitute/dsm/export/WorkflowAndFamilyIdExporter.java index cd7b68164..5d39fc68d 100644 --- a/src/main/java/org/broadinstitute/dsm/export/WorkflowAndFamilyIdExporter.java +++ b/src/main/java/org/broadinstitute/dsm/export/WorkflowAndFamilyIdExporter.java @@ -102,7 +102,7 @@ private void checkWorkflowNamesAndExport(DDPInstance instance, List work WorkflowsEditor editor = new WorkflowsEditor(new ArrayList<>()); try { - Map source = ElasticSearchUtil.getObjectsMap(index, profile.getParticipantGuid(), ESObjectConstants.WORKFLOWS); + Map source = ElasticSearchUtil.getObjectsMap(index, profile.getGuid(), ESObjectConstants.WORKFLOWS); if (source != null && source.containsKey(ESObjectConstants.WORKFLOWS)) { List> workflowListES = (List>) source.get(ESObjectConstants.WORKFLOWS); editor = new WorkflowsEditor(workflowListES); @@ -136,9 +136,9 @@ private void checkWorkflowNamesAndExport(DDPInstance instance, List work try { // Even if workflow list didn't change, let's export so we start with empty list in the ES document. - ElasticSearchUtil.updateRequest(profile.getParticipantGuid(), index, editor.getMapForES()); + ElasticSearchUtil.updateRequest(profile.getGuid(), index, editor.getMapForES()); if (StringUtils.isNotBlank(familyId)) { - ElasticSearchUtil.writeDsmRecord(instance, null, profile.getParticipantGuid(), ESObjectConstants.FAMILY_ID, familyId, null); + ElasticSearchUtil.writeDsmRecord(instance, null, profile.getGuid(), ESObjectConstants.FAMILY_ID, familyId, null); } } catch (Exception e) { logger.error("Error while export ES workflows for participant with guid/altpid: {}, continuing with export", guidOrAltPid, e); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/ESProfile.java b/src/main/java/org/broadinstitute/dsm/model/elastic/ESProfile.java index 4763b3fd3..d5d8ec74a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/ESProfile.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/ESProfile.java @@ -7,16 +7,16 @@ @Data public class ESProfile { @SerializedName(ESObjectConstants.GUID) - private String participantGuid; + private String guid; @SerializedName(ESObjectConstants.HRUID) private String hruid; @SerializedName(ESObjectConstants.LEGACY_ALTPID) - private String participantLegacyAltPid; + private String legacyAltPid; @SerializedName(ESObjectConstants.LEGACY_SHORTID) - private String participantLegacyShortId; + private String legacyShortId; @SerializedName(ESObjectConstants.FIRST_NAME) private String firstName; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index ac517f0b3..a4dc8f222 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -114,8 +114,9 @@ static Map convertToMap(String fieldName, Object fieldValue) { Map finalResult; switch (fieldName) { case "follow_ups": - List> followUps = convertObjectListToMapList(fieldValue); - finalResult = Map.of(underscoresToCamelCase(fieldName), followUps); + finalResult = Map.of(underscoresToCamelCase(fieldName), new Gson().toJson(fieldValue)); +// List> followUps = convertObjectListToMapList(fieldValue); +// finalResult = Map.of(underscoresToCamelCase(fieldName), followUps); break; case "data": Map objectMap = dynamicFieldsSpecialCase(fieldValue); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchParticipantDto.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchParticipantDto.java index 261fb9e38..15879c78d 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchParticipantDto.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchParticipantDto.java @@ -78,12 +78,14 @@ public Optional getComputed() { } public String getParticipantId() { - return getProfile().map(esProfile -> StringUtils.isNotBlank(esProfile.getParticipantGuid()) - ? esProfile.getParticipantGuid() - : esProfile.getParticipantLegacyAltPid()) + return getProfile().map(esProfile -> StringUtils.isNotBlank(esProfile.getGuid()) + ? esProfile.getGuid() + : esProfile.getLegacyAltPid()) .orElse(""); } + public ElasticSearchParticipantDto() {} + private ElasticSearchParticipantDto(ElasticSearchParticipantDto.Builder builder) { this.address = builder.address; this.medicalProviders = builder.medicalProviders; diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java index f55b19ed4..9623d6fa8 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java @@ -221,7 +221,7 @@ List getParticipantIdsFromElasticList(List return elasticSearchParticipantDtos. stream() .flatMap(elasticSearch -> elasticSearch.getProfile().stream()) - .map(ESProfile::getParticipantGuid) + .map(ESProfile::getGuid) .collect(Collectors.toList()); } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java index 0ede74fa2..408ad53bb 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/ExistingRecordPatch.java @@ -15,7 +15,6 @@ import org.broadinstitute.dsm.model.elastic.ESProfile; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import org.broadinstitute.dsm.model.settings.field.FieldSettings; -import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.broadinstitute.dsm.util.NotificationUtil; @@ -117,7 +116,7 @@ private void controlWorkflowByEmail(Patch patch, NameValue nameValue, DDPInstanc writeFamilyMemberWorklow(patch, ddpInstance, profile, pData); } else { Map esMap = ElasticSearchUtil - .getObjectsMap(ddpInstance.getParticipantIndexES(), profile.getParticipantGuid(), + .getObjectsMap(ddpInstance.getParticipantIndexES(), profile.getGuid(), ESObjectConstants.WORKFLOWS); if (Objects.isNull(esMap) || esMap.isEmpty()) return; removeFamilyMemberWorkflowData(ddpInstance, profile, pData, esMap); @@ -136,7 +135,7 @@ private void writeFamilyMemberWorklow(Patch patch, DDPInstance ddpInstance, ESPr if (!patch.getFieldId().contains(org.broadinstitute.dsm.model.participant.data.ParticipantData.FIELD_TYPE_PARTICIPANTS)) return; // Use participant guid here to avoid multiple ES lookups. ElasticSearchUtil.writeWorkflow(WorkflowForES.createInstanceWithStudySpecificData(ddpInstance, - profile.getParticipantGuid(), columnName, columnValue, new WorkflowForES.StudySpecificData( + profile.getGuid(), columnName, columnValue, new WorkflowForES.StudySpecificData( pData.get(FamilyMemberConstants.COLLABORATOR_PARTICIPANT_ID), pData.get(FamilyMemberConstants.FIRSTNAME), pData.get(FamilyMemberConstants.LASTNAME))), false); @@ -159,14 +158,14 @@ private void removeFamilyMemberWorkflowData(DDPInstance ddpInstance, ESProfile p if (startingSize != workflowsList.size()) { esMap.put(ESObjectConstants.WORKFLOWS, workflowsList); // Use participant guid here to avoid another ES lookup. - ElasticSearchUtil.updateRequest(profile.getParticipantGuid(), ddpInstance.getParticipantIndexES(), esMap); + ElasticSearchUtil.updateRequest(profile.getGuid(), ddpInstance.getParticipantIndexES(), esMap); } } private void writeESWorkflowElseTriggerParticipantEvent(Patch patch, DDPInstance ddpInstance, ESProfile profile, NameValue nameValue) { for (Value action : patch.getActions()) { if (hasProfileAndESWorkflowType(profile, action)) { - writeESWorkflow(patch, nameValue, action, ddpInstance, profile.getParticipantGuid()); + writeESWorkflow(patch, nameValue, action, ddpInstance, profile.getGuid()); } else if (EventTypeDao.EVENT.equals(action.getType())) { triggerParticipantEvent(ddpInstance, patch, action); diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantDataPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantDataPatch.java index 3ceb6055d..11284392c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantDataPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/ParticipantDataPatch.java @@ -2,7 +2,6 @@ import org.broadinstitute.dsm.db.ParticipantData; import org.broadinstitute.dsm.db.dao.settings.EventTypeDao; -import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.Value; import org.broadinstitute.dsm.statics.ESObjectConstants; @@ -54,7 +53,7 @@ else if (participantDataId != null) { .orElseThrow(() -> new RuntimeException("Unable to find ES profile for participant: " + patch.getParentId())); for (Value action : patch.getActions()) { if (hasProfileAndESWorkflowType(profile, action)) { - writeESWorkflow(patch, nameValue, action, ddpInstance, profile.getParticipantGuid()); + writeESWorkflow(patch, nameValue, action, ddpInstance, profile.getGuid()); } else if (EventTypeDao.EVENT.equals(action.getType())) { triggerParticipantEvent(ddpInstance, patch, action); diff --git a/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java b/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java index 53e6298be..18c55285f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java +++ b/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java @@ -57,14 +57,14 @@ private boolean extractAndInsertProbandFromESData(DDPInstance instance, ElasticS return esData.getProfile() .map(esProfile -> { - logger.info("Got ES profile of participant: " + esProfile.getParticipantGuid()); + logger.info("Got ES profile of participant: " + esProfile.getGuid()); Map columnsWithDefaultOptions = fieldSettings.getColumnsWithDefaultValues(fieldSettingsDtosByOptionAndInstanceId); Map columnsWithDefaultOptionsFilteredByElasticExportWorkflow = fieldSettings.getColumnsWithDefaultOptionsFilteredByElasticExportWorkflow(fieldSettingsDtosByOptionAndInstanceId); - String participantId = StringUtils.isNotBlank(esProfile.getParticipantLegacyAltPid()) - ? esProfile.getParticipantLegacyAltPid() - : esProfile.getParticipantGuid(); + String participantId = StringUtils.isNotBlank(esProfile.getLegacyAltPid()) + ? esProfile.getLegacyAltPid() + : esProfile.getGuid(); ParticipantData participantData = new ParticipantData(participantDataDao); Optional maybeFamilyIdOfBookmark = bookmarkDao.getBookmarkByInstance(RGP_FAMILY_ID); Map probandDataMap = extractProbandDefaultDataFromParticipantProfile(esData, maybeFamilyIdOfBookmark); @@ -103,7 +103,7 @@ private Map extractProbandDefaultDataFromParticipantProfile(@Non .orElse(""); return esData.getProfile() .map(esProfile -> { - logger.info("Starting extracting data from participant: " + esProfile.getParticipantGuid() + " ES profile"); + logger.info("Starting extracting data from participant: " + esProfile.getGuid() + " ES profile"); String firstName = esProfile.getFirstName(); String lastName = esProfile.getLastName(); long familyId = maybeBookmark @@ -117,7 +117,7 @@ private Map extractProbandDefaultDataFromParticipantProfile(@Non probandMemberDetails.setMobilePhone(mobilePhone); probandMemberDetails.setEmail(email); probandMemberDetails.setApplicant(true); - logger.info("Profile data extracted from participant: " + esProfile.getParticipantGuid() + " ES profile"); + logger.info("Profile data extracted from participant: " + esProfile.getGuid() + " ES profile"); return probandMemberDetails.toMap(); }) .orElse(Map.of()); diff --git a/src/main/java/org/broadinstitute/dsm/route/DashboardRoute.java b/src/main/java/org/broadinstitute/dsm/route/DashboardRoute.java index 4480ca44f..cfda12875 100644 --- a/src/main/java/org/broadinstitute/dsm/route/DashboardRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/DashboardRoute.java @@ -362,7 +362,7 @@ private void countMedicalRecordData(@NonNull List medicalRecordLi if (medicalRecord.isFollowUpRequired()) { incrementCounter(dashboardValuesDetailed, "followupRequiredMedicalRecord"); foundAtPT.add("followupRequiredMedicalRecord"); - if (medicalRecord.getFollowUps() == null || medicalRecord.getFollowUps().size() == 0) { + if (medicalRecord.getFollowUps() == null || medicalRecord.getFollowUps().length == 0) { incrementCounter(dashboardValuesDetailed, "followupNotRequested"); foundAtPT.add("followupNotRequested"); } diff --git a/src/main/java/org/broadinstitute/dsm/route/KitUploadRoute.java b/src/main/java/org/broadinstitute/dsm/route/KitUploadRoute.java index b373e879f..303d78257 100644 --- a/src/main/java/org/broadinstitute/dsm/route/KitUploadRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/KitUploadRoute.java @@ -217,8 +217,8 @@ private void uploadKit(@NonNull DDPInstance ddpInstance, @NonNull KitType kitTyp if (StringUtils.isBlank(kit.getParticipantId())) { ElasticSearchParticipantDto participantByShortId = elasticSearch.getParticipantById(ddpInstance.getParticipantIndexES(), kit.getShortId()); - participantGuid = participantByShortId.getProfile().map(ESProfile::getParticipantGuid).orElse(""); - participantLegacyAltPid = participantByShortId.getProfile().map(ESProfile::getParticipantLegacyAltPid).orElse(""); + participantGuid = participantByShortId.getProfile().map(ESProfile::getGuid).orElse(""); + participantLegacyAltPid = participantByShortId.getProfile().map(ESProfile::getLegacyAltPid).orElse(""); kit.setParticipantId(!participantGuid.isEmpty() ? participantGuid : participantLegacyAltPid); } else { @@ -334,8 +334,8 @@ private void handleKit(@NonNull Connection conn, @NonNull DDPInstance ddpInstanc } } else { - String participantGuid = elasticSearch.getParticipantById(ddpInstance.getParticipantIndexES(), kit.getShortId()).getProfile().map(ESProfile::getParticipantGuid).orElse(""); - String participantLegacyAltPid = elasticSearch.getParticipantById(ddpInstance.getParticipantIndexES(), kit.getShortId()).getProfile().map(ESProfile::getParticipantLegacyAltPid).orElse(""); + String participantGuid = elasticSearch.getParticipantById(ddpInstance.getParticipantIndexES(), kit.getShortId()).getProfile().map(ESProfile::getGuid).orElse(""); + String participantLegacyAltPid = elasticSearch.getParticipantById(ddpInstance.getParticipantIndexES(), kit.getShortId()).getProfile().map(ESProfile::getLegacyAltPid).orElse(""); if (checkAndSetParticipantIdIfKitExists(ddpInstance, conn, kit, participantGuid, participantLegacyAltPid, kitType.getKitTypeId()) && !uploadAnyway) { duplicateKitList.add(kit); } diff --git a/src/main/java/org/broadinstitute/dsm/util/ElasticSearchUtil.java b/src/main/java/org/broadinstitute/dsm/util/ElasticSearchUtil.java index cabf6c2e4..58fd44aec 100644 --- a/src/main/java/org/broadinstitute/dsm/util/ElasticSearchUtil.java +++ b/src/main/java/org/broadinstitute/dsm/util/ElasticSearchUtil.java @@ -396,7 +396,7 @@ public static Map getParticipantAddresses(RestHighLevelClient c ESProfile profile = gson.fromJson(participantJson.get(PROFILE), ESProfile.class); Address gbfAddress = new Address(address.getRecipient(), address.getStreet1(), address.getStreet1(), address.getCity(), address.getState(), address.getZip(), address.getCountry(), address.getPhone()); - addressByParticipant.put(profile.getParticipantGuid(), gbfAddress); + addressByParticipant.put(profile.getGuid(), gbfAddress); } hitNumber++; } @@ -448,7 +448,7 @@ public static void writeWorkflow(RestHighLevelClient client, @NonNull WorkflowFo String participantId = ParticipantUtil.isGuid(ddpParticipantId) ? ddpParticipantId : getParticipantESDataByAltpid(client, index, ddpParticipantId) .getProfile() - .map(ESProfile::getParticipantGuid) + .map(ESProfile::getGuid) .orElse(ddpParticipantId); Map workflowMapES = getObjectsMap(client, index, participantId, ESObjectConstants.WORKFLOWS); @@ -679,7 +679,7 @@ private static void updateRequest(@NonNull String ddpParticipantId, String index RestHighLevelClient client) throws IOException { String participantId = ParticipantUtil.isGuid(ddpParticipantId) ? ddpParticipantId : getParticipantESDataByAltpid(client, index, ddpParticipantId) .getProfile() - .map(ESProfile::getParticipantGuid) + .map(ESProfile::getGuid) .orElse(ddpParticipantId); UpdateRequest updateRequest = new UpdateRequest() .index(index) @@ -747,7 +747,7 @@ public static Optional getParticipantProfileByGuidOrAltPid(String ind Map source = response.getHits().getAt(0).getSourceAsMap(); profile = ElasticSearch.parseSourceMap(source).flatMap(ElasticSearchParticipantDto::getProfile).orElse(null); if (profile != null) { - logger.info("Found ES profile for participant, guid: {} altpid: {}", profile.getParticipantGuid(), profile.getParticipantLegacyAltPid()); + logger.info("Found ES profile for participant, guid: {} altpid: {}", profile.getGuid(), profile.getLegacyAltPid()); } } return Optional.ofNullable(profile); diff --git a/src/test/java/org/broadinstitute/dsm/ElasticSearchTest.java b/src/test/java/org/broadinstitute/dsm/ElasticSearchTest.java index a9ef1c6ae..061ae0ac7 100644 --- a/src/test/java/org/broadinstitute/dsm/ElasticSearchTest.java +++ b/src/test/java/org/broadinstitute/dsm/ElasticSearchTest.java @@ -413,7 +413,7 @@ public void testSearchParticipantById() { ElasticSearchUtil.fetchESDataByParticipantId("participants_structured.rgp.rgp", pIdToFilter, client); fetchedPid = esObject.orElse(new ElasticSearchParticipantDto.Builder().build()) .getProfile() - .map(ESProfile::getParticipantLegacyAltPid) + .map(ESProfile::getLegacyAltPid) .orElse(""); } catch (IOException e) { Assert.fail(); @@ -430,7 +430,7 @@ public void testSearchParticipantByAltpid() { ElasticSearchParticipantDto esObject = ElasticSearchUtil.fetchESDataByAltpid("participants_structured.atcp.atcp", altpid, client); fetchedPid = esObject.getProfile() - .map(ESProfile::getParticipantLegacyAltPid) + .map(ESProfile::getLegacyAltPid) .orElse(""); } catch (IOException e) { Assert.fail(); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchTest.java index a405faf17..845aa48ee 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchTest.java @@ -22,7 +22,7 @@ public void getParticipantIdFromProfile() { .withProfile(profile) .build(); String participantId = elasticSearchParticipantDto.getParticipantId(); - Assert.assertEquals(profile.getParticipantGuid(), participantId); + Assert.assertEquals(profile.getGuid(), participantId); } @Test @@ -32,7 +32,7 @@ public void getParticipantIdFromProfileIfGuidEmpty() { .withProfile(esProfileWithLegacyAltPid) .build(); String participantId = elasticSearchParticipantDto.getParticipantId(); - Assert.assertEquals(esProfileWithLegacyAltPid.getParticipantLegacyAltPid(), participantId); + Assert.assertEquals(esProfileWithLegacyAltPid.getLegacyAltPid(), participantId); } @Test @@ -87,13 +87,13 @@ public void parseSourceMap() { private static ESProfile esProfileGeneratorWithGuid() { ESProfile esProfile = new ESProfile(); - esProfile.setParticipantGuid(ParticipantWrapperTest.randomGuidGenerator()); + esProfile.setGuid(ParticipantWrapperTest.randomGuidGenerator()); return esProfile; } private static ESProfile esProfileGeneratorWithLegacyAltPid() { ESProfile esProfile = new ESProfile(); - esProfile.setParticipantLegacyAltPid(ParticipantWrapperTest.randomLegacyAltPidGenerator()); + esProfile.setLegacyAltPid(ParticipantWrapperTest.randomLegacyAltPidGenerator()); return esProfile; } diff --git a/src/test/java/org/broadinstitute/dsm/model/participant/ParticipantWrapperTest.java b/src/test/java/org/broadinstitute/dsm/model/participant/ParticipantWrapperTest.java index 5d117fb19..1c2f4b649 100644 --- a/src/test/java/org/broadinstitute/dsm/model/participant/ParticipantWrapperTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/participant/ParticipantWrapperTest.java @@ -87,7 +87,7 @@ private static class ElasticSearchTest implements ElasticSearchable { public ElasticSearch getParticipantsWithinRange(String esParticipantsIndex, int from, int to) { List result = Stream.generate(() -> { ESProfile esProfile = new ESProfile(); - esProfile.setParticipantGuid(randomGuidGenerator()); + esProfile.setGuid(randomGuidGenerator()); return new ElasticSearchParticipantDto.Builder() .withProfile(esProfile) .withProxies(generateProxies()) @@ -104,7 +104,7 @@ public ElasticSearch getParticipantsByIds(String esParticipantIndex, List result = new ArrayList<>(); participantIds.forEach(pId -> { ESProfile esProfile = new ESProfile(); - esProfile.setParticipantGuid(pId); + esProfile.setGuid(pId); result.add( new ElasticSearchParticipantDto.Builder() .withProfile(esProfile) diff --git a/src/test/java/org/broadinstitute/dsm/route/KitUploadRouteTest.java b/src/test/java/org/broadinstitute/dsm/route/KitUploadRouteTest.java index 890c9af27..00ff7d47b 100644 --- a/src/test/java/org/broadinstitute/dsm/route/KitUploadRouteTest.java +++ b/src/test/java/org/broadinstitute/dsm/route/KitUploadRouteTest.java @@ -116,7 +116,7 @@ private ParticipantWrapperDto participantFactory(String firstName, String lastNa if (ParticipantUtil.isHruid(shortId)) { esProfile.setHruid(shortId); } else { - esProfile.setParticipantLegacyAltPid(shortId); + esProfile.setLegacyAltPid(shortId); } ElasticSearchParticipantDto elasticSearchParticipantDto = new ElasticSearchParticipantDto.Builder() From 0535e90d545259789b82c2f154f3363e825819f5 Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 24 Nov 2021 16:59:17 +0400 Subject: [PATCH 151/405] DDP-7121 | remove unnecessary else if statement that contained a bug --- .../elastic/export/parse/BaseParser.java | 2 -- .../model/elastic/search/ElasticSearch.java | 34 ++++++++++++++++++- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java index 9f11faeba..6b70bab3d 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java @@ -23,8 +23,6 @@ public Object parse(String value) { result = forBoolean(value); } else if (isDateOrTimeOrDateTime(value)) { result = forDate(value); - } else if (isCollection(value)) { - result = Map.of("type", "nested"); } return result; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java index a1d2b0a82..1979219cd 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java @@ -1,6 +1,7 @@ package org.broadinstitute.dsm.model.elastic.search; import java.io.IOException; +import java.io.StringReader; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -10,7 +11,10 @@ import java.util.Optional; import java.util.stream.Collectors; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; +import com.google.gson.stream.JsonReader; import lombok.Setter; import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.elastic.Util; @@ -31,6 +35,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.broadinstitute.dsm.statics.ESObjectConstants.DSM; + @Setter public class ElasticSearch implements ElasticSearchable { @@ -60,10 +66,36 @@ public long getTotalCount() { public static Optional parseSourceMap(Map sourceMap) { if (sourceMap == null) return Optional.of(new ElasticSearchParticipantDto.Builder().build()); - ElasticSearchParticipantDto elasticSearchParticipantDto = GSON.fromJson(GSON.toJson(sourceMap), ElasticSearchParticipantDto.class); + ElasticSearchParticipantDto elasticSearchParticipantDto = serialize(sourceMap); return Optional.of(elasticSearchParticipantDto); } + private static ElasticSearchParticipantDto serialize(Map sourceMap) { + ElasticSearchParticipantDto elasticSearchParticipantDto; + boolean hasMedicalRecord = false; + boolean hasDSM = sourceMap.containsKey(DSM); + if (hasDSM) + hasMedicalRecord = ((Map) sourceMap.get(DSM)).containsKey("medicalRecords"); + long followUpsCount = 0; + if (hasMedicalRecord) { + followUpsCount = ((List>) (((Map) sourceMap.get(DSM)).get("medicalRecords"))).stream().filter(map -> map.containsKey("followUps")).count(); + } + if (followUpsCount > 0) { + try { + String sourceMapAsString = new ObjectMapper().writeValueAsString(sourceMap); + sourceMapAsString = sourceMapAsString.replace("\"[", "[").replace("]\"", "]").replace("\\\"", "\""); + JsonReader reader = new JsonReader(new StringReader(sourceMapAsString)); + reader.setLenient(true); + elasticSearchParticipantDto = GSON.fromJson(reader, ElasticSearchParticipantDto.class); + } catch (JsonProcessingException e) { + throw new RuntimeException(); + } + } else { + elasticSearchParticipantDto = GSON.fromJson(GSON.toJson(sourceMap), ElasticSearchParticipantDto.class); + } + return elasticSearchParticipantDto; + } + public List parseSourceMaps(SearchHit[] searchHits) { if (Objects.isNull(searchHits)) return Collections.emptyList(); List result = new ArrayList<>(); From 608bad193f0c7e74eb72fcdddf639367d4bf821d Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 24 Nov 2021 18:05:49 +0400 Subject: [PATCH 152/405] DDP-7121 | add anotations, add code to get primary key by reflection --- .../dsm/db/KitRequestShipping.java | 2 +- .../broadinstitute/dsm/db/MedicalRecord.java | 31 ++++++++++++------- .../ddp/participant/ParticipantDataDto.java | 9 +++++- .../export/process/CollectionProcessor.java | 21 +++++++++++-- .../migration/ParticipantDataMigrator.java | 7 +++-- 5 files changed, 52 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java index 73dc24549..0b1257199 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java @@ -38,7 +38,7 @@ @TableName ( name = DBConstants.DDP_KIT_REQUEST, alias = DBConstants.DDP_KIT_REQUEST_ALIAS, - primaryKey = "", + primaryKey = DBConstants.DSM_KIT_REQUEST_ID, columnPrefix = "") public class KitRequestShipping extends KitRequest { diff --git a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java index 6750a99fa..3561dbc34 100644 --- a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java +++ b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java @@ -1,31 +1,40 @@ package org.broadinstitute.dsm.db; -import com.google.common.reflect.TypeToken; +import static org.broadinstitute.ddp.db.TransactionWrapper.inTransaction; + +import java.io.IOException; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import com.google.gson.Gson; import lombok.Data; import lombok.NonNull; -import org.apache.commons.lang3.StringUtils; import org.broadinstitute.ddp.db.SimpleResult; -import org.broadinstitute.ddp.handlers.util.InstitutionDetail; import org.broadinstitute.ddp.handlers.util.MedicalInfo; -import org.broadinstitute.dsm.DSMServer; import org.broadinstitute.dsm.db.structure.ColumnName; import org.broadinstitute.dsm.db.structure.DbDateConversion; import org.broadinstitute.dsm.db.structure.SqlDateConverter; import org.broadinstitute.dsm.db.structure.TableName; import org.broadinstitute.dsm.model.FollowUp; -import org.broadinstitute.dsm.statics.*; +import org.broadinstitute.dsm.statics.DBConstants; +import org.broadinstitute.dsm.statics.QueryExtension; +import org.broadinstitute.dsm.statics.RequestParameter; +import org.broadinstitute.dsm.statics.RoutePath; import org.broadinstitute.dsm.util.DBUtil; import org.broadinstitute.dsm.util.DDPRequestUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.sql.*; -import java.util.*; - -import static org.broadinstitute.ddp.db.TransactionWrapper.inTransaction; - +@TableName ( + name = DBConstants.DDP_MEDICAL_RECORD, + alias = DBConstants.DDP_MEDICAL_RECORD_ALIAS, + primaryKey = DBConstants.MEDICAL_RECORD_ID, + columnPrefix = "") @Data public class MedicalRecord { diff --git a/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java b/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java index 16a6b285e..83ee7537f 100644 --- a/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java +++ b/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java @@ -11,7 +11,14 @@ import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; import org.broadinstitute.dsm.db.structure.ColumnName; - +import org.broadinstitute.dsm.db.structure.TableName; +import org.broadinstitute.dsm.statics.DBConstants; + +@TableName( + name = DBConstants.DDP_PARTICIPANT_DATA, + alias = DBConstants.DDP_PARTICIPANT_DATA_ALIAS, + primaryKey = DBConstants.PARTICIPANT_DATA_ID, + columnPrefix = "") @Setter public class ParticipantDataDto { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java index bfbccb88a..1ac50e8e6 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java @@ -9,6 +9,7 @@ import java.util.Optional; import java.util.function.Predicate; +import org.broadinstitute.dsm.db.structure.TableName; import org.broadinstitute.dsm.model.elastic.ESDsm; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.generate.Collector; @@ -24,6 +25,7 @@ public class CollectionProcessor implements Processor { private String propertyName; private int recordId; private Collector collector; + private String primaryKey; private final Predicate isFieldMatchProperty = field -> propertyName.equals(field.getName()); @@ -55,12 +57,25 @@ private List> extractDataByReflection() { private List> getRecordsByField(Field field) { try { - return Util.convertObjectListToMapList(field.get(esDsm)); + + List objectCollection = (List) field.get(esDsm); + primaryKey = findPrimaryKeyOfObject(objectCollection); + return Util.convertObjectListToMapList(objectCollection); } catch (IllegalAccessException iae) { throw new RuntimeException("error occurred while attempting to get data from ESDsm", iae); } } + private String findPrimaryKeyOfObject(List objectCollection) { + Optional maybeObject = objectCollection.stream().findFirst(); + return maybeObject + .map(o -> { + TableName tableName = o.getClass().getAnnotation(TableName.class); + return tableName != null ? tableName.primaryKey() : ""; + }) + .orElse(""); + } + private List> updateIfExistsOrPut(List> fetchedRecords) { fetchedRecords.stream() .filter(this::isExistingRecord) @@ -75,8 +90,8 @@ private void addNewRecordTo(List> fetchedRecords) { } private boolean isExistingRecord(Map eachRecord) { - if (!eachRecord.containsKey(Util.ID)) return false; - return (double) eachRecord.get(Util.ID) == (double) recordId; + if (!eachRecord.containsKey(Util.underscoresToCamelCase(primaryKey))) return false; + return Double.parseDouble(String.valueOf(eachRecord.get(Util.underscoresToCamelCase(primaryKey)))) == (double) recordId; } private void updateExistingRecord(Map eachRecord) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java index bc7e63890..51363213f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java @@ -11,8 +11,10 @@ import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.elastic.export.ElasticMappingExportAdapter; import org.broadinstitute.dsm.model.elastic.export.RequestPayload; +import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; import org.broadinstitute.dsm.model.elastic.export.generate.MappingGenerator; +import org.broadinstitute.dsm.model.elastic.export.generate.MappingGeneratorFactory; import org.broadinstitute.dsm.model.elastic.export.parse.BaseParser; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; @@ -52,8 +54,9 @@ private void exportMapping() { for (Object jsonData : participantDatas) { NameValue nameValue = new NameValue(DATA_WITH_ALIAS, dataField.get(jsonData)); GeneratorPayload generatorPayload = new GeneratorPayload(nameValue); -// MappingGenerator mappingGenerator = new MappingGenerator(typeParser, generatorPayload); - MappingGenerator mappingGenerator = null; + MappingGenerator mappingGenerator = (MappingGenerator) new MappingGeneratorFactory().make(new BaseGenerator.PropertyInfo("", true)); + mappingGenerator.setParser(typeParser); + mappingGenerator.setPayload(generatorPayload); mapping = mappingGenerator.merge(mapping, mappingGenerator.generate()); } } From 7bffbb3813736986061ff699414b1e3416671eda Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 24 Nov 2021 18:50:58 +0400 Subject: [PATCH 153/405] DDP-7121 | add medical record collection processor to handle followups case --- .../model/elastic/export/ExportFacade.java | 5 +++- .../export/process/CollectionProcessor.java | 12 ++++----- .../process/CollectionProcessorFactory.java | 12 +++++++++ .../MedicalRecordCollectionProcessor.java | 27 +++++++++++++++++++ .../export/process/ProcessorFactory.java | 7 +++++ 5 files changed, 56 insertions(+), 7 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorFactory.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/process/MedicalRecordCollectionProcessor.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/process/ProcessorFactory.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index f21b460d2..89cac364a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -11,6 +11,7 @@ import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; import org.broadinstitute.dsm.model.elastic.export.parse.ValueParser; import org.broadinstitute.dsm.model.elastic.export.process.CollectionProcessor; +import org.broadinstitute.dsm.model.elastic.export.process.MedicalRecordCollectionProcessor; import org.broadinstitute.dsm.model.elastic.export.process.Processor; import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; @@ -72,7 +73,9 @@ private Map processData(ESDsm esDsm) { Map dataToReturn = generator.generate(); logger.info("Processing ES participant data"); if (propertyInfo.isCollection()) { - processor = new CollectionProcessor(esDsm, propertyInfo.getPropertyName(), exportFacadePayload.getRecordId(), +// processor = new CollectionProcessor(esDsm, propertyInfo.getPropertyName(), exportFacadePayload.getRecordId(), +// generator); + processor = new MedicalRecordCollectionProcessor(esDsm, propertyInfo.getPropertyName(), exportFacadePayload.getRecordId(), generator); List> processedData = processor.process(); if (!processedData.isEmpty()) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java index 1ac50e8e6..bd372d6d6 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java @@ -76,7 +76,7 @@ private String findPrimaryKeyOfObject(List objectCollection) { .orElse(""); } - private List> updateIfExistsOrPut(List> fetchedRecords) { + protected List> updateIfExistsOrPut(List> fetchedRecords) { fetchedRecords.stream() .filter(this::isExistingRecord) .findFirst() @@ -84,11 +84,6 @@ private List> updateIfExistsOrPut(List> return fetchedRecords; } - private void addNewRecordTo(List> fetchedRecords) { - logger.info("Adding new record"); - collectEndResult().ifPresent(fetchedRecords::add); - } - private boolean isExistingRecord(Map eachRecord) { if (!eachRecord.containsKey(Util.underscoresToCamelCase(primaryKey))) return false; return Double.parseDouble(String.valueOf(eachRecord.get(Util.underscoresToCamelCase(primaryKey)))) == (double) recordId; @@ -99,6 +94,11 @@ private void updateExistingRecord(Map eachRecord) { collectEndResult().ifPresent(eachRecord::putAll); } + private void addNewRecordTo(List> fetchedRecords) { + logger.info("Adding new record"); + collectEndResult().ifPresent(fetchedRecords::add); + } + private Optional> collectEndResult() { return ((List>) collector.collect()) .stream() diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorFactory.java new file mode 100644 index 000000000..f69ee1677 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorFactory.java @@ -0,0 +1,12 @@ +package org.broadinstitute.dsm.model.elastic.export.process; + +import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; + +public class CollectionProcessorFactory implements ProcessorFactory { + + + @Override + public Processor make(BaseGenerator.PropertyInfo propertyInfo) { + return null; + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/MedicalRecordCollectionProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/MedicalRecordCollectionProcessor.java new file mode 100644 index 000000000..33b4f33cd --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/MedicalRecordCollectionProcessor.java @@ -0,0 +1,27 @@ +package org.broadinstitute.dsm.model.elastic.export.process; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import com.google.gson.Gson; +import org.broadinstitute.dsm.model.elastic.ESDsm; +import org.broadinstitute.dsm.model.elastic.export.generate.Collector; + +public class MedicalRecordCollectionProcessor extends CollectionProcessor { + public MedicalRecordCollectionProcessor(ESDsm esDsm, String propertyName, int recordId, + Collector collector) { + super(esDsm, propertyName, recordId, collector); + } + + @Override + protected List> updateIfExistsOrPut(List> fetchedRecords) { + for (Map medicalRecord: fetchedRecords) { + Object followUps = medicalRecord.get("followUps"); + if (!Objects.isNull(followUps)) { + medicalRecord.put("followUps", new Gson().toJson(followUps)); + } + } + return super.updateIfExistsOrPut(fetchedRecords); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/ProcessorFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/ProcessorFactory.java new file mode 100644 index 000000000..369b73cc5 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/ProcessorFactory.java @@ -0,0 +1,7 @@ +package org.broadinstitute.dsm.model.elastic.export.process; + +import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; + +public interface ProcessorFactory { + Processor make(BaseGenerator.PropertyInfo propertyInfo); +} From 4155ddebb1fd431325013957d70de584b430a044 Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 24 Nov 2021 19:34:51 +0400 Subject: [PATCH 154/405] DDP-7121 | introduced BaseProcessor abstraction --- .../model/elastic/export/ExportFacade.java | 16 +++---- .../elastic/export/process/BaseProcessor.java | 42 +++++++++++++++++++ .../export/process/CollectionProcessor.java | 15 +++---- .../process/CollectionProcessorFactory.java | 11 +++-- .../MedicalRecordCollectionProcessor.java | 4 ++ 5 files changed, 68 insertions(+), 20 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/process/BaseProcessor.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 89cac364a..b9a2cf3a6 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -10,9 +10,7 @@ import org.broadinstitute.dsm.model.elastic.export.generate.*; import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; import org.broadinstitute.dsm.model.elastic.export.parse.ValueParser; -import org.broadinstitute.dsm.model.elastic.export.process.CollectionProcessor; -import org.broadinstitute.dsm.model.elastic.export.process.MedicalRecordCollectionProcessor; -import org.broadinstitute.dsm.model.elastic.export.process.Processor; +import org.broadinstitute.dsm.model.elastic.export.process.*; import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchable; @@ -28,7 +26,7 @@ public class ExportFacade { BaseGenerator generator; ElasticSearchable searchable; private ExportFacadePayload exportFacadePayload; - Processor processor; + BaseProcessor processor; public ExportFacade(ExportFacadePayload exportFacadePayload) { this.exportFacadePayload = Objects.requireNonNull(exportFacadePayload); @@ -73,10 +71,12 @@ private Map processData(ESDsm esDsm) { Map dataToReturn = generator.generate(); logger.info("Processing ES participant data"); if (propertyInfo.isCollection()) { -// processor = new CollectionProcessor(esDsm, propertyInfo.getPropertyName(), exportFacadePayload.getRecordId(), -// generator); - processor = new MedicalRecordCollectionProcessor(esDsm, propertyInfo.getPropertyName(), exportFacadePayload.getRecordId(), - generator); + CollectionProcessorFactory collectionProcessorFactory = new CollectionProcessorFactory(); + processor = collectionProcessorFactory.make(propertyInfo); + processor.setEsDsm(esDsm); + processor.setPropertyName(propertyInfo.getPropertyName()); + processor.setRecordId(exportFacadePayload.getRecordId()); + processor.setCollector(generator); List> processedData = processor.process(); if (!processedData.isEmpty()) { dataToReturn = Map.of(MappingGenerator.DSM_OBJECT, Map.of(propertyInfo.getPropertyName(), processedData)); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/BaseProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/BaseProcessor.java new file mode 100644 index 000000000..33a48c410 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/BaseProcessor.java @@ -0,0 +1,42 @@ +package org.broadinstitute.dsm.model.elastic.export.process; + +import org.broadinstitute.dsm.model.elastic.ESDsm; +import org.broadinstitute.dsm.model.elastic.export.generate.Collector; + +import java.util.Objects; + +public abstract class BaseProcessor implements Processor { + + + protected ESDsm esDsm; + protected String propertyName; + protected int recordId; + protected Collector collector; + + public BaseProcessor(ESDsm esDsm, String propertyName, int recordId, Collector collector) { + this.esDsm = Objects.requireNonNull(esDsm); + this.propertyName = Objects.requireNonNull(propertyName); + this.recordId = recordId; + this.collector = collector; + } + + public BaseProcessor() { + + } + + public void setEsDsm(ESDsm esDsm) { + this.esDsm = esDsm; + } + + public void setPropertyName(String propertyName) { + this.propertyName = propertyName; + } + + public void setRecordId(int recordId) { + this.recordId = recordId; + } + + public void setCollector(Collector collector) { + this.collector = collector; + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java index bd372d6d6..5d134b63e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java @@ -17,23 +17,20 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class CollectionProcessor implements Processor { +public class CollectionProcessor extends BaseProcessor { private static final Logger logger = LoggerFactory.getLogger(CollectionProcessor.class); - private ESDsm esDsm; - private String propertyName; - private int recordId; - private Collector collector; private String primaryKey; private final Predicate isFieldMatchProperty = field -> propertyName.equals(field.getName()); public CollectionProcessor(ESDsm esDsm, String propertyName, int recordId, Collector collector) { - this.esDsm = Objects.requireNonNull(esDsm); - this.propertyName = Objects.requireNonNull(propertyName); - this.recordId = recordId; - this.collector = collector; + super(esDsm, propertyName, recordId, collector); + } + + public CollectionProcessor() { + } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorFactory.java index f69ee1677..b77d18c51 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorFactory.java @@ -4,9 +4,14 @@ public class CollectionProcessorFactory implements ProcessorFactory { - @Override - public Processor make(BaseGenerator.PropertyInfo propertyInfo) { - return null; + public BaseProcessor make(BaseGenerator.PropertyInfo propertyInfo) { + BaseProcessor processor = new CollectionProcessor(); + if (propertyInfo.isCollection()) { + if ("followUps".equals(propertyInfo.getFieldName())) { + processor = new MedicalRecordCollectionProcessor(); + } + } + return processor; } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/MedicalRecordCollectionProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/MedicalRecordCollectionProcessor.java index 33b4f33cd..6c8603001 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/MedicalRecordCollectionProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/MedicalRecordCollectionProcessor.java @@ -14,6 +14,10 @@ public MedicalRecordCollectionProcessor(ESDsm esDsm, String propertyName, int re super(esDsm, propertyName, recordId, collector); } + public MedicalRecordCollectionProcessor() { + + } + @Override protected List> updateIfExistsOrPut(List> fetchedRecords) { for (Map medicalRecord: fetchedRecords) { From a32e68aced14a71e80c526cc09707d09d47f41d9 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 25 Nov 2021 10:44:22 +0400 Subject: [PATCH 155/405] DDP-7121 | clean up of serialization --- .../export/process/CollectionProcessor.java | 8 ++- .../process/CollectionProcessorFactory.java | 3 +- .../model/elastic/search/ElasticSearch.java | 68 +++++++++++++------ 3 files changed, 56 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java index 5d134b63e..f1c316058 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java @@ -64,6 +64,7 @@ private List> getRecordsByField(Field field) { } private String findPrimaryKeyOfObject(List objectCollection) { + if (Objects.isNull(objectCollection)) return ""; Optional maybeObject = objectCollection.stream().findFirst(); return maybeObject .map(o -> { @@ -82,8 +83,11 @@ protected List> updateIfExistsOrPut(List } private boolean isExistingRecord(Map eachRecord) { - if (!eachRecord.containsKey(Util.underscoresToCamelCase(primaryKey))) return false; - return Double.parseDouble(String.valueOf(eachRecord.get(Util.underscoresToCamelCase(primaryKey)))) == (double) recordId; + if (!eachRecord.containsKey(Util.underscoresToCamelCase(primaryKey)) || !eachRecord.containsKey(Util.ID)) return false; + double id = eachRecord.containsKey(Util.ID) + ? Double.parseDouble(String.valueOf(eachRecord.get(Util.ID))) + : Double.parseDouble(String.valueOf(eachRecord.get(Util.underscoresToCamelCase(primaryKey)))); + return id == (double) recordId; } private void updateExistingRecord(Map eachRecord) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorFactory.java index b77d18c51..86b7c569b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorFactory.java @@ -1,6 +1,7 @@ package org.broadinstitute.dsm.model.elastic.export.process; import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; +import org.broadinstitute.dsm.statics.ESObjectConstants; public class CollectionProcessorFactory implements ProcessorFactory { @@ -8,7 +9,7 @@ public class CollectionProcessorFactory implements ProcessorFactory { public BaseProcessor make(BaseGenerator.PropertyInfo propertyInfo) { BaseProcessor processor = new CollectionProcessor(); if (propertyInfo.isCollection()) { - if ("followUps".equals(propertyInfo.getFieldName())) { + if (ESObjectConstants.MEDICAL_RECORDS.equals(propertyInfo.getPropertyName())) { processor = new MedicalRecordCollectionProcessor(); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java index 1979219cd..de4e01cba 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java @@ -17,6 +17,7 @@ import com.google.gson.stream.JsonReader; import lombok.Setter; import org.apache.commons.lang3.StringUtils; +import org.broadinstitute.dsm.db.MedicalRecord; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.broadinstitute.dsm.util.ParticipantUtil; @@ -71,29 +72,56 @@ public static Optional parseSourceMap(Map sourceMap) { - ElasticSearchParticipantDto elasticSearchParticipantDto; - boolean hasMedicalRecord = false; - boolean hasDSM = sourceMap.containsKey(DSM); - if (hasDSM) - hasMedicalRecord = ((Map) sourceMap.get(DSM)).containsKey("medicalRecords"); - long followUpsCount = 0; - if (hasMedicalRecord) { - followUpsCount = ((List>) (((Map) sourceMap.get(DSM)).get("medicalRecords"))).stream().filter(map -> map.containsKey("followUps")).count(); - } - if (followUpsCount > 0) { - try { - String sourceMapAsString = new ObjectMapper().writeValueAsString(sourceMap); - sourceMapAsString = sourceMapAsString.replace("\"[", "[").replace("]\"", "]").replace("\\\"", "\""); - JsonReader reader = new JsonReader(new StringReader(sourceMapAsString)); - reader.setLenient(true); - elasticSearchParticipantDto = GSON.fromJson(reader, ElasticSearchParticipantDto.class); - } catch (JsonProcessingException e) { - throw new RuntimeException(); + ElasticSearchParticipantDto elasticSearchParticipantDto = null; + Object dsm = sourceMap.get(DSM); + if (!Objects.isNull(dsm)) { + List> medicalRecords = (List>)((Map) sourceMap.get(DSM)).get("medicalRecords"); + if (!Objects.isNull(medicalRecords)) { + boolean hasFollowUps = medicalRecords.stream() + .anyMatch(medicalRecord -> StringUtils.isNotBlank((String) medicalRecord.get("followUps"))); + if (hasFollowUps) { + String sourceMapAsString = null; + try { + sourceMapAsString = new ObjectMapper().writeValueAsString(sourceMap); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + sourceMapAsString = sourceMapAsString.replace("\"[", "[").replace("]\"", "]").replace("\\\"", "\""); + JsonReader reader = new JsonReader(new StringReader(sourceMapAsString)); + reader.setLenient(true); + elasticSearchParticipantDto = GSON.fromJson(reader, ElasticSearchParticipantDto.class); + } else { + elasticSearchParticipantDto = GSON.fromJson(GSON.toJson(sourceMap), ElasticSearchParticipantDto.class); + } } - } else { - elasticSearchParticipantDto = GSON.fromJson(GSON.toJson(sourceMap), ElasticSearchParticipantDto.class); } return elasticSearchParticipantDto; +// boolean hasMedicalRecord = false; +// boolean hasDSM = sourceMap.containsKey(DSM); +// if (hasDSM) +// hasMedicalRecord = ((Map) sourceMap.get(DSM)).containsKey("medicalRecords"); +// long followUpsCount = 0; +// if (hasMedicalRecord) { +// followUpsCount = ((List>) (((Map) sourceMap.get(DSM)).get("medicalRecords"))).stream().filter(map -> map.containsKey("followUps")).count(); +// } +// if (followUpsCount > 0) { +// try { +// String sourceMapAsString = new ObjectMapper().writeValueAsString(sourceMap); +// sourceMapAsString = sourceMapAsString.replace("\"[", "[").replace("]\"", "]").replace("\\\"", "\""); +// JsonReader reader = new JsonReader(new StringReader(sourceMapAsString)); +// reader.setLenient(true); +// elasticSearchParticipantDto = GSON.fromJson(reader, ElasticSearchParticipantDto.class); +// } catch (JsonProcessingException e) { +// throw new RuntimeException(); +// } +// } else { +// elasticSearchParticipantDto = GSON.fromJson(GSON.toJson(sourceMap), ElasticSearchParticipantDto.class); +// } +// return elasticSearchParticipantDto; + } + + private static String extractListFromString(String listInString) { + return listInString.replace("\"[", "[").replace("]\"", "]").replace("\\\"", "\""); } public List parseSourceMaps(SearchHit[] searchHits) { From e9b153caec6352e2af4202b00d61ccaef3d31ec2 Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 25 Nov 2021 11:14:12 +0400 Subject: [PATCH 156/405] DDP-7121 | use ObjectMapper.convertValue() instead of GSON-s functionality --- .../model/elastic/search/ElasticSearch.java | 46 +++++++++++++------ 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java index de4e01cba..bced40a7c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java @@ -12,12 +12,14 @@ import java.util.stream.Collectors; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; import com.google.gson.stream.JsonReader; import lombok.Setter; import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.MedicalRecord; +import org.broadinstitute.dsm.model.FollowUp; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.broadinstitute.dsm.util.ParticipantUtil; @@ -77,23 +79,39 @@ private static ElasticSearchParticipantDto serialize(Map sourceM if (!Objects.isNull(dsm)) { List> medicalRecords = (List>)((Map) sourceMap.get(DSM)).get("medicalRecords"); if (!Objects.isNull(medicalRecords)) { - boolean hasFollowUps = medicalRecords.stream() - .anyMatch(medicalRecord -> StringUtils.isNotBlank((String) medicalRecord.get("followUps"))); - if (hasFollowUps) { - String sourceMapAsString = null; - try { - sourceMapAsString = new ObjectMapper().writeValueAsString(sourceMap); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); + for (Map medicalRecord : medicalRecords) { + if (StringUtils.isNotBlank((String) medicalRecord.get("followUps"))) { + String followUps = (String) medicalRecord.get("followUps"); + try { + medicalRecord.put("followUps", new ObjectMapper().readValue(followUps, new TypeReference>(){})); + } catch (IOException e) { + throw new RuntimeException(e); + } } - sourceMapAsString = sourceMapAsString.replace("\"[", "[").replace("]\"", "]").replace("\\\"", "\""); - JsonReader reader = new JsonReader(new StringReader(sourceMapAsString)); - reader.setLenient(true); - elasticSearchParticipantDto = GSON.fromJson(reader, ElasticSearchParticipantDto.class); - } else { - elasticSearchParticipantDto = GSON.fromJson(GSON.toJson(sourceMap), ElasticSearchParticipantDto.class); } + + +// if (hasFollowUps) { +// String sourceMapAsString = null; +// try { +// sourceMapAsString = new ObjectMapper().writeValueAsString(sourceMap); +// } catch (JsonProcessingException e) { +// throw new RuntimeException(e); +// } +// sourceMapAsString = sourceMapAsString.replace("\"[", "[").replace("]\"", "]").replace("\\\"", "\""); +// JsonReader reader = new JsonReader(new StringReader(sourceMapAsString)); +// reader.setLenient(true); +// elasticSearchParticipantDto = GSON.fromJson(reader, ElasticSearchParticipantDto.class); +// } else { +// elasticSearchParticipantDto = GSON.fromJson(GSON.toJson(sourceMap), ElasticSearchParticipantDto.class); +// } + elasticSearchParticipantDto = new ObjectMapper().convertValue(sourceMap, ElasticSearchParticipantDto.class); + + } else { + elasticSearchParticipantDto = new ObjectMapper().convertValue(sourceMap, ElasticSearchParticipantDto.class); } + } else { + elasticSearchParticipantDto = new ObjectMapper().convertValue(sourceMap, ElasticSearchParticipantDto.class); } return elasticSearchParticipantDto; // boolean hasMedicalRecord = false; From 83dae0f9f1761a9111c4da4514e3db84c0adb781 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 25 Nov 2021 11:45:08 +0400 Subject: [PATCH 157/405] DDP-7121 | change to jackson --- .../dsm/db/KitRequestShipping.java | 57 ++++++++++--------- .../broadinstitute/dsm/db/MedicalRecord.java | 14 ++++- .../dsm/db/OncHistoryDetail.java | 6 +- .../broadinstitute/dsm/db/Participant.java | 43 +++++++------- .../broadinstitute/dsm/model/KitRequest.java | 2 + .../dsm/model/elastic/ESDsm.java | 6 +- .../dsm/route/DashboardRoute.java | 18 +++--- .../dsm/route/KitExpressRoute.java | 4 +- .../dsm/route/ParticipantExitRoute.java | 2 +- .../org/broadinstitute/dsm/util/PDFAudit.java | 4 +- 10 files changed, 89 insertions(+), 67 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java index 0b1257199..9056de818 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java @@ -2,6 +2,7 @@ import com.easypost.exception.EasyPostException; import com.easypost.model.*; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.Data; import lombok.NonNull; import org.apache.commons.lang3.StringUtils; @@ -9,7 +10,6 @@ import org.broadinstitute.ddp.db.TransactionWrapper; import org.broadinstitute.dsm.DSMServer; import org.broadinstitute.dsm.db.structure.ColumnName; -import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.db.structure.DbDateConversion; import org.broadinstitute.dsm.db.structure.SqlDateConverter; import org.broadinstitute.dsm.db.structure.TableName; @@ -40,6 +40,7 @@ alias = DBConstants.DDP_KIT_REQUEST_ALIAS, primaryKey = DBConstants.DSM_KIT_REQUEST_ID, columnPrefix = "") +@JsonIgnoreProperties(ignoreUnknown = true) public class KitRequestShipping extends KitRequest { private static final Logger logger = LoggerFactory.getLogger(KitRequestShipping.class); @@ -142,65 +143,64 @@ public class KitRequestShipping extends KitRequest { private static final String SEARCH_MF_BAR = "MF_BAR"; @ColumnName(DBConstants.DSM_KIT_ID) - private final String dsmKitId; + private String dsmKitId; @ColumnName(DBConstants.LABEL_URL_TO) - private final String labelUrlTo; + private String labelUrlTo; @ColumnName(DBConstants.LABEL_URL_RETURN) - private final String labelUrlReturn; + private String labelUrlReturn; @ColumnName (DBConstants.DSM_TRACKING_TO) - private final String trackingNumberTo; + private String trackingNumberTo; @ColumnName (DBConstants.DSM_TRACKING_RETURN) - private final String trackingNumberReturn; + private String trackingNumberReturn; @ColumnName (DBConstants.TRACKING_ID) private String scannedTrackingNumber; - private final String trackingUrlTo; - private final String trackingUrlReturn; - private final String collaboratorParticipantId; + private String trackingUrlTo; + private String trackingUrlReturn; + private String collaboratorParticipantId; @ColumnName (DBConstants.BSP_COLLABORATOR_PARTICIPANT_ID) - private final String bspCollaboratorSampleId; - - private final String easypostAddressId; - private final String realm; + private String bspCollaboratorSampleId; + private String easypostAddressId; + private String realm; @ColumnName (DBConstants.KIT_TYPE_NAME) - private final String kitType; + private String kitTypeName; @ColumnName (DBConstants.DEACTIVATION_REASON) - private final String deactivationReason; + private String deactivationReason; @ColumnName (DBConstants.KIT_LABEL) - private final String kitLabel; + private String kitLabel; @ColumnName (DBConstants.KIT_TEST_RESULT) private String testResult; @ColumnName (DBConstants.DSM_SCAN_DATE) @DbDateConversion(SqlDateConverter.EPOCH) - private final long scanDate; + private long scanDate; @ColumnName (DBConstants.DSM_RECEIVE_DATE) @DbDateConversion(SqlDateConverter.EPOCH) - private final long receiveDate; + private long receiveDate; @ColumnName (DBConstants.DSM_DEACTIVATED_DATE) @DbDateConversion(SqlDateConverter.EPOCH) - private final long deactivatedDate; + private long deactivatedDate; @ColumnName (DBConstants.EXPRESS) - private final boolean express; + private boolean express; @ColumnName (DBConstants.EASYPOST_TO_ID) - private final String easypostToId; - private final long labelTriggeredDate; + private String easypostToId; + private long labelTriggeredDate; @ColumnName (DBConstants.NO_RETURN) - private final boolean noReturn; + private boolean noReturn; @ColumnName (DBConstants.ERROR) private boolean error; @@ -231,9 +231,12 @@ public class KitRequestShipping extends KitRequest { @ColumnName (DBConstants.UPLOAD_REASON) private String uploadReason; - public KitRequestShipping(String collaboratorParticipantId, String kitType, String dsmKitRequestId, long scanDate, boolean error, long receiveDate, long deactivatedDate, String testResult, + + public KitRequestShipping() {} + + public KitRequestShipping(String collaboratorParticipantId, String kitTypeName, String dsmKitRequestId, long scanDate, boolean error, long receiveDate, long deactivatedDate, String testResult, String upsTrackingStatus, String upsReturnStatus, String externalOrderStatus, String externalOrderNumber, long externalOrderDate, boolean careEvolve, String uploadReason) { - this(null, collaboratorParticipantId, null, null, null, kitType, dsmKitRequestId, null, null, null, + this(null, collaboratorParticipantId, null, null, null, kitTypeName, dsmKitRequestId, null, null, null, null, null, null, null, scanDate, error, null, receiveDate, null, deactivatedDate, null, null, false, null, 0, null, externalOrderNumber, false, externalOrderStatus, null, testResult, upsTrackingStatus, upsReturnStatus, externalOrderDate, false, uploadReason, null, null, null); @@ -255,7 +258,7 @@ public KitRequestShipping(String dsmKitRequestId, String dsmKitId, String easypo // shippingId = ddp_label !!! public KitRequestShipping(String participantId, String collaboratorParticipantId, String bspCollaboratorSampleId, String shippingId, String realm, - String kitType, String dsmKitRequestId, String dsmKitId, String labelUrlTo, String labelUrlReturn, + String kitTypeName, String dsmKitRequestId, String dsmKitId, String labelUrlTo, String labelUrlReturn, String trackingNumberTo, String trackingNumberReturn, String trackingUrlTo, String trackingUrlReturn, long scanDate, boolean error, String message, long receiveDate, String easypostAddressId, long deactivatedDate, String deactivationReason, @@ -267,7 +270,7 @@ public KitRequestShipping(String participantId, String collaboratorParticipantId this.collaboratorParticipantId = collaboratorParticipantId; this.bspCollaboratorSampleId = bspCollaboratorSampleId; this.realm = realm; - this.kitType = kitType; + this.kitTypeName = kitTypeName; this.dsmKitId = dsmKitId; this.labelUrlTo = labelUrlTo; this.labelUrlReturn = labelUrlReturn; diff --git a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java index 3561dbc34..73e887ad6 100644 --- a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java +++ b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java @@ -11,6 +11,8 @@ import java.util.List; import java.util.Map; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import com.google.gson.Gson; import lombok.Data; import lombok.NonNull; @@ -36,6 +38,7 @@ primaryKey = DBConstants.MEDICAL_RECORD_ID, columnPrefix = "") @Data +@JsonIgnoreProperties(ignoreUnknown = true) public class MedicalRecord { private static final Logger logger = LoggerFactory.getLogger(MedicalRecord.class); @@ -260,7 +263,7 @@ public class MedicalRecord { primaryKey = DBConstants.MEDICAL_RECORD_ID, columnPrefix = "") @ColumnName (DBConstants.MR_UNABLE_OBTAIN_TEXT) - private String mrUnableToObtainText; + private String unableObtainText; @TableName ( name = DBConstants.DDP_MEDICAL_RECORD, @@ -270,6 +273,11 @@ public class MedicalRecord { @ColumnName (DBConstants.FOLLOWUP_REQUIRED) private boolean followUpRequired; + @JsonProperty("followupRequired") + public boolean isFollowUpRequired() { + return followUpRequired; + } + @TableName ( name = DBConstants.DDP_MEDICAL_RECORD, alias = DBConstants.DDP_MEDICAL_RECORD_ALIAS, @@ -354,7 +362,7 @@ public MedicalRecord(String medicalRecordId, String institutionId, String ddpIns String mrProblemText, boolean unableObtain, boolean duplicate, boolean international, boolean crRequired, String pathologyPresent, String mrNotes, boolean reviewMedicalRecord, FollowUp[] followUps, boolean followUpRequired, String followUpRequiredText, String additionalValues, - String mrUnableToObtainText, String ddpParticipantId) { + String unableObtainText, String ddpParticipantId) { this.medicalRecordId = medicalRecordId; this.institutionId = institutionId; this.ddpInstitutionId = ddpInstitutionId; @@ -388,7 +396,7 @@ public MedicalRecord(String medicalRecordId, String institutionId, String ddpIns this.followUpRequired = followUpRequired; this.followUpRequiredText = followUpRequiredText; this.additionalValues = additionalValues; - this.mrUnableToObtainText = mrUnableToObtainText; + this.unableObtainText = unableObtainText; this.ddpParticipantId = ddpParticipantId; } diff --git a/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java b/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java index ead8bbae9..8657b1bde 100644 --- a/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java +++ b/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java @@ -1,5 +1,6 @@ package org.broadinstitute.dsm.db; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.Data; import lombok.NonNull; import org.apache.commons.lang3.StringUtils; @@ -29,6 +30,7 @@ alias = DBConstants.DDP_ONC_HISTORY_DETAIL_ALIAS, primaryKey = DBConstants.ONC_HISTORY_DETAIL_ID, columnPrefix = "") +@JsonIgnoreProperties(ignoreUnknown = true) public class OncHistoryDetail { private static final Logger logger = LoggerFactory.getLogger(OncHistoryDetail.class); @@ -80,7 +82,7 @@ public class OncHistoryDetail { private String oncHistoryDetailId; @ColumnName(DBConstants.MEDICAL_RECORD_ID) - private final String medicalRecordId; + private String medicalRecordId; @ColumnName (DBConstants.DATE_PX) private String datePx; @@ -173,6 +175,8 @@ public class OncHistoryDetail { private List tissues; + public OncHistoryDetail() {} + public OncHistoryDetail(String oncHistoryDetailId, String medicalRecordId, String datePx, String typePx, String locationPx, String histology, String accessionNumber, String facility, String phone, String fax, String notes, String request, String faxSent, diff --git a/src/main/java/org/broadinstitute/dsm/db/Participant.java b/src/main/java/org/broadinstitute/dsm/db/Participant.java index 2c30de9a5..e6d56e0db 100644 --- a/src/main/java/org/broadinstitute/dsm/db/Participant.java +++ b/src/main/java/org/broadinstitute/dsm/db/Participant.java @@ -15,7 +15,6 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.*; -import java.util.stream.Collectors; import static org.broadinstitute.ddp.db.TransactionWrapper.inTransaction; @@ -35,10 +34,10 @@ public class Participant { "WHERE realm.instance_name = ? "; @ColumnName(DBConstants.PARTICIPANT_ID) - private final String participantId; + private String participantId; @ColumnName(DBConstants.DDP_PARTICIPANT_ID) - private final String ddpParticipantId; + private String ddpParticipantId; @TableName ( name = DBConstants.DDP_PARTICIPANT, @@ -46,7 +45,7 @@ public class Participant { primaryKey = DBConstants.PARTICIPANT_ID, columnPrefix = "") @ColumnName (DBConstants.ASSIGNEE_ID_MR) - private final String assigneeMr; + private String assigneeIdMr; @TableName ( name = DBConstants.DDP_PARTICIPANT, @@ -54,8 +53,8 @@ public class Participant { primaryKey = DBConstants.PARTICIPANT_ID, columnPrefix = "") @ColumnName (DBConstants.ASSIGNEE_ID_TISSUE) - private final String assigneeTissue; - private final String realm; + private String assigneeIdTissue; + private String realm; @TableName ( name = DBConstants.DDP_ONC_HISTORY, @@ -63,7 +62,7 @@ public class Participant { primaryKey = DBConstants.PARTICIPANT_ID, columnPrefix = "") @ColumnName (DBConstants.ONC_HISTORY_CREATED) - private final String createdOncHistory; + private String createdOncHistory; @TableName ( name = DBConstants.DDP_ONC_HISTORY, @@ -71,7 +70,7 @@ public class Participant { primaryKey = DBConstants.PARTICIPANT_ID, columnPrefix = "") @ColumnName (DBConstants.ONC_HISTORY_REVIEWED) - private final String reviewedOncHistory; + private String reviewedOncHistory; @TableName ( name = DBConstants.DDP_PARTICIPANT_RECORD, @@ -79,7 +78,7 @@ public class Participant { primaryKey = DBConstants.PARTICIPANT_ID, columnPrefix = "") @ColumnName (DBConstants.CR_SENT) - private final String paperCRSent; + private String paperCRSent; @TableName ( name = DBConstants.DDP_PARTICIPANT_RECORD, @@ -87,7 +86,7 @@ public class Participant { primaryKey = DBConstants.PARTICIPANT_ID, columnPrefix = "") @ColumnName (DBConstants.CR_RECEIVED) - private final String paperCRReceived; + private String paperCRReceived; @TableName ( name = DBConstants.DDP_PARTICIPANT_RECORD, @@ -95,7 +94,7 @@ public class Participant { primaryKey = DBConstants.PARTICIPANT_ID, columnPrefix = "") @ColumnName (DBConstants.NOTES) - private final String ptNotes; + private String ptNotes; @TableName ( name = DBConstants.DDP_PARTICIPANT_RECORD, @@ -103,7 +102,7 @@ public class Participant { primaryKey = DBConstants.PARTICIPANT_ID, columnPrefix = "") @ColumnName (DBConstants.MINIMAL_MR) - private final boolean minimalMR; + private boolean minimalMr; @TableName ( name = DBConstants.DDP_PARTICIPANT_RECORD, @@ -111,7 +110,7 @@ public class Participant { primaryKey = DBConstants.PARTICIPANT_ID, columnPrefix = "") @ColumnName (DBConstants.ABSTRACTION_READY) - private final boolean abstractionReady; + private boolean abstractionReady; @TableName ( name = DBConstants.DDP_PARTICIPANT_RECORD, @@ -119,7 +118,7 @@ public class Participant { primaryKey = DBConstants.PARTICIPANT_ID, columnPrefix = "") @ColumnName (DBConstants.ADDITIONAL_VALUES) - private final String additionalValues; + private String additionalValuesJson; @TableName ( name = DBConstants.DDP_PARTICIPANT_EXIT, @@ -127,24 +126,26 @@ public class Participant { primaryKey = DBConstants.PARTICIPANT_ID, columnPrefix = "") @ColumnName (DBConstants.EXIT_DATE) - private final long exitDate; + private long exitDate; - public Participant(String participantId, String ddpParticipantId, String assigneeMr, String assigneeTissue, String instanceName, + public Participant() {} + + public Participant(String participantId, String ddpParticipantId, String assigneeIdMr, String assigneeIdTissue, String instanceName, String createdOncHistory, String reviewedOncHistory, String paperCRSent, String paperCRReceived, String ptNotes, - boolean minimalMR, boolean abstractionReady, String additionalValues, long exitDate) { + boolean minimalMr, boolean abstractionReady, String additionalValuesJson, long exitDate) { this.participantId = participantId; this.ddpParticipantId = ddpParticipantId; - this.assigneeMr = assigneeMr; - this.assigneeTissue = assigneeTissue; + this.assigneeIdMr = assigneeIdMr; + this.assigneeIdTissue = assigneeIdTissue; this.realm = instanceName; this.createdOncHistory = createdOncHistory; this.reviewedOncHistory = reviewedOncHistory; this.paperCRSent = paperCRSent; this.paperCRReceived = paperCRReceived; this.ptNotes = ptNotes; - this.minimalMR = minimalMR; + this.minimalMr = minimalMr; this.abstractionReady = abstractionReady; - this.additionalValues = additionalValues; + this.additionalValuesJson = additionalValuesJson; this.exitDate = exitDate; } diff --git a/src/main/java/org/broadinstitute/dsm/model/KitRequest.java b/src/main/java/org/broadinstitute/dsm/model/KitRequest.java index 59600aa12..2314c4bb2 100644 --- a/src/main/java/org/broadinstitute/dsm/model/KitRequest.java +++ b/src/main/java/org/broadinstitute/dsm/model/KitRequest.java @@ -34,6 +34,8 @@ public class KitRequest { @DbDateConversion(SqlDateConverter.EPOCH) private Long externalOrderDate; + public KitRequest() {} + public KitRequest(String participantId, String shortId, DDPParticipant participant, String externalOrderNumber) { this(null, participantId, shortId, null, externalOrderNumber, participant, null, null, null); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java index a7fbb735b..5b261a8e2 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java @@ -5,7 +5,9 @@ import com.google.gson.annotations.SerializedName; import lombok.Getter; +import org.broadinstitute.dsm.db.KitRequestShipping; import org.broadinstitute.dsm.db.MedicalRecord; +import org.broadinstitute.dsm.db.OncHistory; import org.broadinstitute.dsm.db.OncHistoryDetail; import org.broadinstitute.dsm.db.Participant; import org.broadinstitute.dsm.db.ParticipantData; @@ -45,7 +47,7 @@ public class ESDsm { List medicalRecords; @SerializedName(ESObjectConstants.ONC_HISTORY_DETAIL_RECORDS) - List oncHistoryDetailRecords; + List oncHistoryDetails; @SerializedName(ESObjectConstants.PARTICIPANT_DATA) List participantData; @@ -53,5 +55,7 @@ public class ESDsm { @SerializedName(ESObjectConstants.PARTICIPANT) Participant participant; + List kitRequestShipping; + OncHistory oncHistory; } diff --git a/src/main/java/org/broadinstitute/dsm/route/DashboardRoute.java b/src/main/java/org/broadinstitute/dsm/route/DashboardRoute.java index cfda12875..eea895d22 100644 --- a/src/main/java/org/broadinstitute/dsm/route/DashboardRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/DashboardRoute.java @@ -301,7 +301,7 @@ public DashboardInformation getMedicalRecordDashboard(@NonNull long start, @NonN } if (wrapper.getParticipant() != null) { - if (wrapper.getParticipant().isMinimalMR()) { + if (wrapper.getParticipant().isMinimalMr()) { incrementCounter(dashboardValues, "minimalMR"); } } @@ -499,22 +499,22 @@ private void countKits(@NonNull List kits, @NonNull Set kitTypes = org.broadinstitute.dsm.model.KitType.getKitLookup(); - String key = kitRequest.getKitType() + "_" + ddpInstance.getDdpInstanceId(); + String key = kitRequest.getKitTypeName() + "_" + ddpInstance.getDdpInstanceId(); KitType kitType = kitTypes.get(key); Map carrierServiceTypes = KitRequestSettings.getKitRequestSettings(ddpInstance.getDdpInstanceId()); @@ -136,7 +136,7 @@ private EasypostLabelRate getRateForOvernightExpress(@NonNull String kitRequestI DDPInstance ddpInstance = DDPInstance.getDDPInstance(kitRequest.getRealm()); HashMap kitTypes = org.broadinstitute.dsm.model.KitType.getKitLookup(); - String key = kitRequest.getKitType() + "_" + ddpInstance.getDdpInstanceId(); + String key = kitRequest.getKitTypeName() + "_" + ddpInstance.getDdpInstanceId(); KitType kitType = kitTypes.get(key); Map carrierServiceTypes = KitRequestSettings.getKitRequestSettings(ddpInstance.getDdpInstanceId()); diff --git a/src/main/java/org/broadinstitute/dsm/route/ParticipantExitRoute.java b/src/main/java/org/broadinstitute/dsm/route/ParticipantExitRoute.java index fd8c647a2..74b02ec1f 100644 --- a/src/main/java/org/broadinstitute/dsm/route/ParticipantExitRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/ParticipantExitRoute.java @@ -62,7 +62,7 @@ public Object processRequest(Request request, Response response, String userId) for (KitRequestShipping kit : kitRequests) { if (kit.getScanDate() != 0 && kit.getReceiveDate() == 0) { String discardId = KitDiscard.addKitToDiscard(kit.getDsmKitRequestId(), KitDiscard.HOLD); - kitsNeedAction.add(new KitDiscard(discardId, kit.getKitType(), KitDiscard.HOLD)); + kitsNeedAction.add(new KitDiscard(discardId, kit.getKitTypeName(), KitDiscard.HOLD)); } else { //refund label of kits which are not sent yet diff --git a/src/main/java/org/broadinstitute/dsm/util/PDFAudit.java b/src/main/java/org/broadinstitute/dsm/util/PDFAudit.java index 3e61f079f..d59bc0aa9 100644 --- a/src/main/java/org/broadinstitute/dsm/util/PDFAudit.java +++ b/src/main/java/org/broadinstitute/dsm/util/PDFAudit.java @@ -26,14 +26,14 @@ public void checkAndSavePDF() { // ddp with 'standard' consent and release pdfs DDPInstance instance = DDPInstance.getDDPInstanceWithRole(request.getRealm(), DBConstants.PDF_DOWNLOAD_CONSENT); //the role for release will get checked in makePDF if (instance != null && StringUtils.isNotBlank(instance.getBaseUrl()) && instance.isHasRole()) { - DDPRequestUtil.makeStandardPDF(instance, request.getParticipantId(), request.getCreatedBy(), request.getKitType()); + DDPRequestUtil.makeStandardPDF(instance, request.getParticipantId(), request.getCreatedBy(), request.getKitTypeName()); } else { // ddp without 'standard' consent and release pdfs instance = DDPInstance.getDDPInstanceWithRole(request.getRealm(), DBConstants.PDF_DOWNLOAD); //the role for release will get checked in makePDF if (instance != null && StringUtils.isNotBlank(instance.getBaseUrl()) && StringUtils.isNotBlank(instance.getParticipantIndexES()) && instance.isHasRole()) { - DDPRequestUtil.makeNonStandardPDF(instance, request.getParticipantId(), request.getCreatedBy(), request.getKitType()); + DDPRequestUtil.makeNonStandardPDF(instance, request.getParticipantId(), request.getCreatedBy(), request.getKitTypeName()); } } newAuditKit = Math.max(newAuditKit, Integer.parseInt(request.getDsmKitRequestId())); From 5f5a171bfdde916560dbdcf17e57a777ff8efff2 Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 25 Nov 2021 17:22:04 +0400 Subject: [PATCH 158/405] DDP-7121 | modify FollowUp and Participant for ObjectMapper in order to parse them --- .../java/org/broadinstitute/dsm/db/Participant.java | 2 ++ .../java/org/broadinstitute/dsm/model/FollowUp.java | 11 +++++++---- .../elastic/export/process/CollectionProcessor.java | 3 +-- .../dsm/model/elastic/search/ElasticSearch.java | 2 +- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/Participant.java b/src/main/java/org/broadinstitute/dsm/db/Participant.java index e6d56e0db..d02ecbd51 100644 --- a/src/main/java/org/broadinstitute/dsm/db/Participant.java +++ b/src/main/java/org/broadinstitute/dsm/db/Participant.java @@ -1,5 +1,6 @@ package org.broadinstitute.dsm.db; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.Data; import lombok.NonNull; import org.apache.commons.lang3.StringUtils; @@ -19,6 +20,7 @@ import static org.broadinstitute.ddp.db.TransactionWrapper.inTransaction; @Data +@JsonIgnoreProperties(ignoreUnknown = true) public class Participant { private static final Logger logger = LoggerFactory.getLogger(Participant.class); diff --git a/src/main/java/org/broadinstitute/dsm/model/FollowUp.java b/src/main/java/org/broadinstitute/dsm/model/FollowUp.java index dcc09833e..28b0ed831 100644 --- a/src/main/java/org/broadinstitute/dsm/model/FollowUp.java +++ b/src/main/java/org/broadinstitute/dsm/model/FollowUp.java @@ -7,16 +7,16 @@ public class FollowUp { @ColumnName("fReceived") - private String fReceived; + public String fReceived; @ColumnName("fRequest1") - private String fRequest1; + public String fRequest1; @ColumnName("fRequest2") - private String fRequest2; + public String fRequest2; @ColumnName("fRequest3") - private String fRequest3; + public String fRequest3; public FollowUp(String fRequest1, String fRequest2, String fRequest3, String fReceived) { this.fReceived = fReceived; @@ -24,5 +24,8 @@ public FollowUp(String fRequest1, String fRequest2, String fRequest3, String fRe this.fRequest2 = fRequest2; this.fRequest3 = fRequest3; + } + public FollowUp() { + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java index f1c316058..6fb983155 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java @@ -54,7 +54,6 @@ private List> extractDataByReflection() { private List> getRecordsByField(Field field) { try { - List objectCollection = (List) field.get(esDsm); primaryKey = findPrimaryKeyOfObject(objectCollection); return Util.convertObjectListToMapList(objectCollection); @@ -83,7 +82,7 @@ protected List> updateIfExistsOrPut(List } private boolean isExistingRecord(Map eachRecord) { - if (!eachRecord.containsKey(Util.underscoresToCamelCase(primaryKey)) || !eachRecord.containsKey(Util.ID)) return false; + if (!eachRecord.containsKey(Util.underscoresToCamelCase(primaryKey)) && !eachRecord.containsKey(Util.ID)) return false; double id = eachRecord.containsKey(Util.ID) ? Double.parseDouble(String.valueOf(eachRecord.get(Util.ID))) : Double.parseDouble(String.valueOf(eachRecord.get(Util.underscoresToCamelCase(primaryKey)))); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java index bced40a7c..b25bb60d0 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java @@ -83,7 +83,7 @@ private static ElasticSearchParticipantDto serialize(Map sourceM if (StringUtils.isNotBlank((String) medicalRecord.get("followUps"))) { String followUps = (String) medicalRecord.get("followUps"); try { - medicalRecord.put("followUps", new ObjectMapper().readValue(followUps, new TypeReference>(){})); + medicalRecord.put("followUps", new ObjectMapper().readValue(followUps, new TypeReference>>(){})); } catch (IOException e) { throw new RuntimeException(e); } From 6d17c3acd6de99629f6b45a57c0aba26cd46350b Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 25 Nov 2021 17:47:25 +0400 Subject: [PATCH 159/405] DDP-7121 | change mapping generator for json to make property level --- .../org/broadinstitute/dsm/db/MedicalRecord.java | 6 +++--- .../elastic/export/generate/MappingGenerator.java | 2 +- .../export/generate/MappingGeneratorTest.java | 14 ++++++++++++++ 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java index 73e887ad6..8bb316e92 100644 --- a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java +++ b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java @@ -332,7 +332,7 @@ public boolean isFollowUpRequired() { primaryKey = DBConstants.MEDICAL_RECORD_ID, columnPrefix = "") @ColumnName (DBConstants.ADDITIONAL_VALUES) - private String additionalValues; + private String additionalValuesJson; private boolean reviewMedicalRecord; @@ -361,7 +361,7 @@ public MedicalRecord(String medicalRecordId, String institutionId, String ddpIns String mrReceived, String mrDocument, String mrDocumentFileNames, boolean mrProblem, String mrProblemText, boolean unableObtain, boolean duplicate, boolean international, boolean crRequired, String pathologyPresent, String mrNotes, boolean reviewMedicalRecord, - FollowUp[] followUps, boolean followUpRequired, String followUpRequiredText, String additionalValues, + FollowUp[] followUps, boolean followUpRequired, String followUpRequiredText, String additionalValuesJson, String unableObtainText, String ddpParticipantId) { this.medicalRecordId = medicalRecordId; this.institutionId = institutionId; @@ -395,7 +395,7 @@ public MedicalRecord(String medicalRecordId, String institutionId, String ddpIns this.followUps = followUps; this.followUpRequired = followUpRequired; this.followUpRequiredText = followUpRequiredText; - this.additionalValues = additionalValues; + this.additionalValuesJson = additionalValuesJson; this.unableObtainText = unableObtainText; this.ddpParticipantId = ddpParticipantId; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java index 52519ea78..cdd9301cd 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java @@ -41,7 +41,7 @@ protected Map parseJson() { Object eachType = parser.parse(String.valueOf(entry.getValue())); resultMap.put(Util.underscoresToCamelCase(entry.getKey()), eachType); } - return resultMap; + return Map.of(Util.underscoresToCamelCase(getDBElement().getColumnName()), resultMap); } @Override diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java index 0c090d12f..a5a2aee6a 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java @@ -116,6 +116,20 @@ public void merge() { Assert.assertFalse(Objects.isNull(value)); } + @Test + public void parseJson() { + NameValue nameValue = new NameValue("m.additionalValuesJson", "{\"DDP_INSTANCE\": \"TEST\", \"DDP_INSTANCE1\": \"TEST1\"}"); + GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, 0); + TypeParser parser = new TypeParser(); + MappingGenerator mappingGenerator = new CollectionMappingGenerator(parser, generatorPayload); + Map parseJson = mappingGenerator.parseJson(); + Map additionalValuesJson = (Map)parseJson.get("additionalValuesJson"); + Assert.assertNotNull(additionalValuesJson); + Assert.assertEquals(TypeParser.TEXT_KEYWORD_MAPPING, additionalValuesJson.get(Util.underscoresToCamelCase("DDP_INSTANCE"))); + Assert.assertEquals(TypeParser.TEXT_KEYWORD_MAPPING, additionalValuesJson.get(Util.underscoresToCamelCase("DDP_INSTANCE1"))); + } + + private String extractDeepestLeveleValue(Map objectMap) { return (String) ((Map) From 60ad08e787c93a7edcd1f4a50e5d6d0ccba609f8 Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 25 Nov 2021 18:12:44 +0400 Subject: [PATCH 160/405] DDP-7121 | write test for parseJson(), start modifying the code based on new design, mapping update is neede while using export script --- .../export/generate/MappingGenerator.java | 2 +- .../model/elastic/migration/BaseMigrator.java | 9 +++++ .../export/generate/MappingGeneratorTest.java | 34 ++++++++++--------- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java index cdd9301cd..f0e7656bd 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java @@ -41,7 +41,7 @@ protected Map parseJson() { Object eachType = parser.parse(String.valueOf(entry.getValue())); resultMap.put(Util.underscoresToCamelCase(entry.getKey()), eachType); } - return Map.of(Util.underscoresToCamelCase(getDBElement().getColumnName()), resultMap); + return Map.of(Util.underscoresToCamelCase(getDBElement().getColumnName()), Map.of(PROPERTIES, resultMap)); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseMigrator.java index 44c28ccbf..428f186b6 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseMigrator.java @@ -4,6 +4,7 @@ import org.broadinstitute.dsm.model.elastic.export.BaseExporter; import org.broadinstitute.dsm.model.elastic.export.Exportable; +import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; import org.broadinstitute.dsm.model.elastic.export.generate.Generator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -17,6 +18,7 @@ public abstract class BaseMigrator extends BaseExporter implements Generator { protected final BulkExportFacade bulkExportFacade; protected final String realm; protected final String index; + BaseGenerator baseGenerator; public BaseMigrator(String index, String realm, String object) { bulkExportFacade = new BulkExportFacade(index); @@ -25,6 +27,12 @@ public BaseMigrator(String index, String realm, String object) { this.object = object; } + public void setBaseGenerator(BaseGenerator baseGenerator) { + this.baseGenerator = baseGenerator; + } + + + protected void fillBulkRequestWithTransformedMap(Map participantRecords) { logger.info("filling bulk request with participants and their details"); for (Map.Entry entry: participantRecords.entrySet()) { @@ -45,6 +53,7 @@ protected void fillBulkRequestWithTransformedMap(Map participant @Override public void export() { + Object mapping = baseGenerator.collect(); fillBulkRequestWithTransformedMap(getDataByRealm()); bulkExportFacade.executeBulkUpsert(); logger.info("finished migrating data to ES."); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java index a5a2aee6a..c9227eea7 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java @@ -14,6 +14,8 @@ import java.util.Map; import java.util.Objects; +import static org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator.PROPERTIES; + public class MappingGeneratorTest { @@ -24,7 +26,7 @@ public void generateTextType() { 0 ); Map objectMap = TestCollectionMappingGenerator.of(generatorPayload).generate(); - Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), BaseGenerator.PROPERTIES); + Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), PROPERTIES); String type = extractDeepestLeveleValue(objectMap); Assert.assertEquals("text", type); } @@ -36,7 +38,7 @@ public void generateTextTypeWithFields() { 0 ); Map objectMap = TestCollectionMappingGenerator.of(generatorPayload).generate(); - Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), BaseGenerator.PROPERTIES); + Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), PROPERTIES); String type = extractKeywordType(objectMap); Assert.assertEquals("keyword", type); } @@ -48,7 +50,7 @@ public void generateBooleanType() { 0 ); Map objectMap = TestCollectionMappingGenerator.of(generatorPayload).generate(); - Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), BaseGenerator.PROPERTIES); + Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), PROPERTIES); String type = extractDeepestLeveleValue(objectMap); Assert.assertEquals("boolean", type); } @@ -60,7 +62,7 @@ public void generateDateType() { 0 ); Map objectMap = TestCollectionMappingGenerator.of(generatorPayload).generate(); - Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), BaseGenerator.PROPERTIES); + Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), PROPERTIES); String type = extractDeepestLeveleValue(objectMap); Assert.assertEquals("date", type); } @@ -85,12 +87,12 @@ public void generateMapping() { Map resultMap = generator.generate(); Map dsmLevelProperty = Map.of(generator.getOuterPropertyByAlias().getPropertyName(), Map.of( MappingGenerator.TYPE, MappingGenerator.NESTED, - MappingGenerator.PROPERTIES, Map.of(Util.underscoresToCamelCase(TestPatchUtil.MEDICAL_RECORD_COLUMN), Map.of(MappingGenerator.TYPE, "date"), + PROPERTIES, Map.of(Util.underscoresToCamelCase(TestPatchUtil.MEDICAL_RECORD_COLUMN), Map.of(MappingGenerator.TYPE, "date"), Util.ID, Map.of(MappingGenerator.TYPE, MappingGenerator.TYPE_KEYWORD) ))); - Map dsmLevelProperties = Map.of(MappingGenerator.PROPERTIES, dsmLevelProperty); + Map dsmLevelProperties = Map.of(PROPERTIES, dsmLevelProperty); Map dsmLevel = Map.of(MappingGenerator.DSM_OBJECT, dsmLevelProperties); - Map topLevel = Map.of(MappingGenerator.PROPERTIES, dsmLevel); + Map topLevel = Map.of(PROPERTIES, dsmLevel); Assert.assertTrue(topLevel.equals(resultMap)); } @@ -107,11 +109,11 @@ public void merge() { mappingGenerator = new CollectionMappingGenerator(parser, generatorPayload2); base = mappingGenerator.merge(base, mappingGenerator.generate()); Object value = ((Map) ((Map) ((Map) ((Map) ((Map) base - .get(MappingGenerator.PROPERTIES)) + .get(PROPERTIES)) .get(MappingGenerator.DSM_OBJECT)) - .get(MappingGenerator.PROPERTIES)) + .get(PROPERTIES)) .get(mappingGenerator.getOuterPropertyByAlias().getPropertyName())) - .get(MappingGenerator.PROPERTIES)) + .get(PROPERTIES)) .get(Util.underscoresToCamelCase("DDP_INSTANCE1")); Assert.assertFalse(Objects.isNull(value)); } @@ -125,8 +127,8 @@ public void parseJson() { Map parseJson = mappingGenerator.parseJson(); Map additionalValuesJson = (Map)parseJson.get("additionalValuesJson"); Assert.assertNotNull(additionalValuesJson); - Assert.assertEquals(TypeParser.TEXT_KEYWORD_MAPPING, additionalValuesJson.get(Util.underscoresToCamelCase("DDP_INSTANCE"))); - Assert.assertEquals(TypeParser.TEXT_KEYWORD_MAPPING, additionalValuesJson.get(Util.underscoresToCamelCase("DDP_INSTANCE1"))); + Assert.assertEquals(TypeParser.TEXT_KEYWORD_MAPPING,((Map) additionalValuesJson.get(PROPERTIES)).get(Util.underscoresToCamelCase("DDP_INSTANCE"))); + Assert.assertEquals(TypeParser.TEXT_KEYWORD_MAPPING,((Map) additionalValuesJson.get(PROPERTIES)).get(Util.underscoresToCamelCase("DDP_INSTANCE1"))); } @@ -135,7 +137,7 @@ private String extractDeepestLeveleValue(Map objectMap) { ((Map) ((Map) getMedicalRecordProperty(objectMap) - .get(BaseGenerator.PROPERTIES)) + .get(PROPERTIES)) .get("medicalRecordColumn")) .get("type"); } @@ -147,7 +149,7 @@ private String extractKeywordType(Map objectMap) { ((Map) ((Map) getMedicalRecordProperty(objectMap) - .get(BaseGenerator.PROPERTIES)) + .get(PROPERTIES)) .get("medicalRecordColumn")) .get("fields")) .get("keyword")) @@ -159,9 +161,9 @@ private Map getMedicalRecordProperty(Map objectMap) { ((Map) ((Map) ((Map) objectMap - .get(BaseGenerator.PROPERTIES)) + .get(PROPERTIES)) .get(BaseGenerator.DSM_OBJECT)) - .get(BaseGenerator.PROPERTIES)) + .get(PROPERTIES)) .get("medicalRecords"); } From 828027b7fbf65fc94ef484241ec3a3708588a539 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 29 Nov 2021 12:20:14 +0400 Subject: [PATCH 161/405] DDP-7121 | changes for json type values --- .../export/generate/BaseGenerator.java | 15 ++++--- .../generate/CollectionMappingGenerator.java | 2 +- .../generate/CollectionSourceGenerator.java | 2 +- .../export/generate/MappingGenerator.java | 19 +++++---- .../generate/SingleMappingGenerator.java | 2 +- .../generate/SingleSourceGenerator.java | 2 +- .../migration/BaseCollectionMigrator.java | 42 +++++++++++++++++++ .../model/elastic/migration/BaseMigrator.java | 13 +++--- .../migration/MedicalRecordMigrator.java | 2 + .../export/generate/MappingGeneratorTest.java | 8 ++++ 10 files changed, 83 insertions(+), 24 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java index 80edeebec..6f0cef521 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java @@ -14,7 +14,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public abstract class BaseGenerator implements Generator, Collector { +public abstract class BaseGenerator implements Generator, Collector, Merger { private static final Logger logger = LoggerFactory.getLogger(BaseGenerator.class); @@ -56,9 +56,9 @@ protected NameValue getNameValue() { //wrap Util.getDBElement in protected method so that we can override it in testing class for tests protected DBElement getDBElement() { - if (dbElement == null) - dbElement = Util.getDBElement(getNameValue().getName()); - return dbElement; +// if (dbElement == null) +// dbElement = Util.getDBElement(getNameValue().getName()); + return Util.getDBElement(getNameValue().getName()); } protected PropertyInfo getOuterPropertyByAlias() { @@ -89,7 +89,12 @@ protected Object parseSingleElement() { protected abstract Object getElement(Object type); - protected abstract Object construct(); + public abstract Object construct(); + + @Override + public Map merge(Map base, Map toMerge) { + return Map.of(); + } public static class PropertyInfo { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java index 8b086ea8e..1918acbfe 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java @@ -23,7 +23,7 @@ protected Map getElement(Object type) { } @Override - protected Map construct() { + public Map construct() { return new HashMap<>(Map.of(TYPE, NESTED, PROPERTIES, collect())); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java index cd9b71a6b..08fe4d48a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java @@ -23,7 +23,7 @@ public CollectionSourceGenerator() { } @Override - protected Object construct() { + public Object construct() { logger.info("Constructing nested data"); Map collectionMap = new HashMap<>(); collectionMap.put(Util.ID, generatorPayload.getRecordId()); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java index f0e7656bd..f5c51171c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java @@ -8,7 +8,7 @@ import java.util.HashMap; import java.util.Map; -abstract public class MappingGenerator extends BaseGenerator implements Merger { +abstract public class MappingGenerator extends BaseGenerator { private static final Logger logger = LoggerFactory.getLogger(MappingGenerator.class); @@ -25,8 +25,12 @@ public MappingGenerator() {} @Override public Map generate() { logger.info("preparing mapping to upsert"); + return getCompleteMap(construct()); + } + + public Map getCompleteMap(Object propertyMap) { String propertyName = getOuterPropertyByAlias().getPropertyName(); - Map objectLevel = Map.of(propertyName, construct()); + Map objectLevel = Map.of(propertyName, propertyMap); Map dsmLevelProperties = Map.of(PROPERTIES, objectLevel); Map> dsmLevel = Map.of(DSM_OBJECT, dsmLevelProperties); return Map.of(PROPERTIES, dsmLevel); @@ -46,18 +50,19 @@ protected Map parseJson() { @Override public Map merge(Map base, Map toMerge) { - base.putIfAbsent(PROPERTIES, toMerge.get(PROPERTIES)); + String propertyName = Util.underscoresToCamelCase(getDBElement().getColumnName()); + base.putIfAbsent(propertyName, getFieldLevel(toMerge)); getFieldLevel(base).putAll(getFieldLevel(toMerge)); return base; } private Map getFieldLevel(Map topLevel) { - String propertyName = getOuterPropertyByAlias().getPropertyName(); - return ((Map)((Map)((Map)((Map)((Map)topLevel.get(PROPERTIES)) - .get(DSM_OBJECT)) + String propertyName = Util.underscoresToCamelCase(getDBElement().getColumnName()); + Map fieldLevel = (Map) ((Map) ((Map) topLevel .get(PROPERTIES)) .get(propertyName)) - .get(PROPERTIES)); + .get(PROPERTIES); + return fieldLevel != null ? fieldLevel : topLevel; } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java index 9f73aab5e..cab8c5b9d 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java @@ -20,7 +20,7 @@ protected Map getElement(Object type) { } @Override - protected Map construct() { + public Map construct() { return new HashMap<>(Map.of(PROPERTIES, collect())); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleSourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleSourceGenerator.java index 4cc4e8379..735b9ffc5 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleSourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleSourceGenerator.java @@ -21,7 +21,7 @@ public SingleSourceGenerator() { } @Override - protected Object construct() { + public Object construct() { return parseJsonValuesToObject(); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java index 627408087..358ca4dec 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java @@ -5,14 +5,25 @@ import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; +import org.broadinstitute.dsm.db.structure.ColumnName; import org.broadinstitute.dsm.db.structure.TableName; +import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.model.elastic.export.ElasticMappingExportAdapter; +import org.broadinstitute.dsm.model.elastic.export.RequestPayload; +import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; +import org.broadinstitute.dsm.model.elastic.export.generate.CollectionMappingGenerator; +import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; +import org.broadinstitute.dsm.model.elastic.export.generate.MappingGenerator; +import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; import org.broadinstitute.dsm.statics.ESObjectConstants; public abstract class BaseCollectionMigrator extends BaseMigrator { + private final MappingGenerator collectionMappingGenerator = new CollectionMappingGenerator(); protected List> transformedList; protected Set primaryKeys; + private Map endResult = new HashMap<>(); public BaseCollectionMigrator(String index, String realm, String object) { super(index, realm, object); @@ -29,6 +40,29 @@ protected void transformObject(Object object) { List objects = (List) object; Optional maybeObject = objects.stream().findFirst(); maybeObject.ifPresent(this::collectPrimaryKeys); + collectionMappingGenerator.setParser(new TypeParser()); + for (Object o: objects) { + Class clazz = o.getClass(); + Field[] declaredFields = clazz.getDeclaredFields(); + TableName table = clazz.getAnnotation(TableName.class); + for (Field field: declaredFields) { + ColumnName columnName = field.getAnnotation(ColumnName.class); + if (columnName == null) continue; + field.setAccessible(true); + String fieldName = field.getName(); + String fieldNameWithAlias = table.alias() + "." + fieldName; + try { + Object fieldValue = field.get(o); + if (fieldValue == null) continue; + NameValue nameValue = new NameValue(fieldNameWithAlias, fieldValue); + GeneratorPayload generatorPayload = new GeneratorPayload(nameValue); + collectionMappingGenerator.setPayload(generatorPayload); + endResult = collectionMappingGenerator.merge(endResult, (Map) collectionMappingGenerator.construct()); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + } + } transformedList = Util.transformObjectCollectionToCollectionMap((List) object); setPrimaryId(); } @@ -113,4 +147,12 @@ private Optional getPrimaryKey(Map map) { private void putPrimaryId(Map map, String outerKey) { map.put(Util.ID, map.get(outerKey)); } + + @Override + protected void exportMap() { + ElasticMappingExportAdapter elasticMappingExportAdapter = new ElasticMappingExportAdapter(); + elasticMappingExportAdapter.setRequestPayload(new RequestPayload(index)); + elasticMappingExportAdapter.setSource(collectionMappingGenerator.getCompleteMap(endResult)); + elasticMappingExportAdapter.export(); + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseMigrator.java index 428f186b6..29faaa2ad 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseMigrator.java @@ -3,7 +3,9 @@ import java.util.Map; import org.broadinstitute.dsm.model.elastic.export.BaseExporter; +import org.broadinstitute.dsm.model.elastic.export.ElasticMappingExportAdapter; import org.broadinstitute.dsm.model.elastic.export.Exportable; +import org.broadinstitute.dsm.model.elastic.export.RequestPayload; import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; import org.broadinstitute.dsm.model.elastic.export.generate.Generator; import org.slf4j.Logger; @@ -18,7 +20,6 @@ public abstract class BaseMigrator extends BaseExporter implements Generator { protected final BulkExportFacade bulkExportFacade; protected final String realm; protected final String index; - BaseGenerator baseGenerator; public BaseMigrator(String index, String realm, String object) { bulkExportFacade = new BulkExportFacade(index); @@ -27,12 +28,6 @@ public BaseMigrator(String index, String realm, String object) { this.object = object; } - public void setBaseGenerator(BaseGenerator baseGenerator) { - this.baseGenerator = baseGenerator; - } - - - protected void fillBulkRequestWithTransformedMap(Map participantRecords) { logger.info("filling bulk request with participants and their details"); for (Map.Entry entry: participantRecords.entrySet()) { @@ -51,10 +46,12 @@ protected void fillBulkRequestWithTransformedMap(Map participant protected abstract Map getDataByRealm(); + protected void exportMap() {} + @Override public void export() { - Object mapping = baseGenerator.collect(); fillBulkRequestWithTransformedMap(getDataByRealm()); + exportMap(); bulkExportFacade.executeBulkUpsert(); logger.info("finished migrating data to ES."); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigrator.java index c135fab59..bae55acb9 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigrator.java @@ -3,8 +3,10 @@ import java.util.Map; import org.broadinstitute.dsm.db.MedicalRecord; +import org.broadinstitute.dsm.model.elastic.export.generate.CollectionMappingGenerator; import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; +import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; import org.broadinstitute.dsm.statics.ESObjectConstants; public class MedicalRecordMigrator extends BaseCollectionMigrator { diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java index c9227eea7..0eaca236a 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java @@ -115,7 +115,15 @@ public void merge() { .get(mappingGenerator.getOuterPropertyByAlias().getPropertyName())) .get(PROPERTIES)) .get(Util.underscoresToCamelCase("DDP_INSTANCE1")); + Object value2 = ((Map) ((Map) ((Map) ((Map) ((Map) base + .get(PROPERTIES)) + .get(MappingGenerator.DSM_OBJECT)) + .get(PROPERTIES)) + .get(mappingGenerator.getOuterPropertyByAlias().getPropertyName())) + .get(PROPERTIES)) + .get(Util.underscoresToCamelCase("DDP_INSTANCE")); Assert.assertFalse(Objects.isNull(value)); + Assert.assertFalse(Objects.isNull(value2)); } @Test From a7705b6a713e4a4aa8ef7fff321e538c41745a68 Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 29 Nov 2021 13:18:25 +0400 Subject: [PATCH 162/405] DDP-7121 | tweak getFieldLevel() of MappingGenerator --- .../export/generate/MappingGenerator.java | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java index f5c51171c..ca7e3460f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java @@ -51,18 +51,27 @@ protected Map parseJson() { @Override public Map merge(Map base, Map toMerge) { String propertyName = Util.underscoresToCamelCase(getDBElement().getColumnName()); - base.putIfAbsent(propertyName, getFieldLevel(toMerge)); +// base.putIfAbsent(propertyName, toMerge); + + + + //dsm -> properties -> medicalRecords -> properties -> mrRecordId: {}, ragaca: {}, additionalValuesjson: { properties: {key: {type = ...} getFieldLevel(base).putAll(getFieldLevel(toMerge)); return base; } private Map getFieldLevel(Map topLevel) { + if (topLevel.isEmpty()) { + return topLevel; + } String propertyName = Util.underscoresToCamelCase(getDBElement().getColumnName()); - Map fieldLevel = (Map) ((Map) ((Map) topLevel - .get(PROPERTIES)) - .get(propertyName)) - .get(PROPERTIES); - return fieldLevel != null ? fieldLevel : topLevel; +// Map outerPropertyMap = (Map) topLevel.get(PROPERTIES); + Map innerPropertyMap = (Map) topLevel.get(propertyName); + Map innerProperty = (Map) innerPropertyMap.get(PROPERTIES); + if (innerProperty != null) + return innerProperty; + else + return innerPropertyMap; } } From b98ef8654e27a18b9f4698afe900678510ee0752 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 29 Nov 2021 17:15:36 +0400 Subject: [PATCH 163/405] DDP-7121 | wrapping map.of to hashmap constructor for mutability --- .../dsm/model/elastic/Util.java | 6 +-- .../model/elastic/export/ExportFacade.java | 5 +- .../export/generate/BaseGenerator.java | 13 +++++- .../generate/CollectionMappingGenerator.java | 11 +++-- .../generate/CollectionSourceGenerator.java | 4 +- .../export/generate/MappingGenerator.java | 14 +++--- .../export/generate/NullObjectGenerator.java | 3 +- .../generate/SingleMappingGenerator.java | 2 +- .../generate/SingleSourceGenerator.java | 3 +- .../export/generate/SourceGenerator.java | 4 +- .../elastic/export/parse/TypeParser.java | 11 +++-- .../migration/BaseCollectionMigrator.java | 46 +++++++++++++++++-- .../elastic/migration/BaseSingleMigrator.java | 3 +- .../model/elastic/search/ElasticSearch.java | 4 +- .../export/generate/MappingGeneratorTest.java | 6 +-- 15 files changed, 99 insertions(+), 36 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index a4dc8f222..721354bd0 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -114,7 +114,7 @@ static Map convertToMap(String fieldName, Object fieldValue) { Map finalResult; switch (fieldName) { case "follow_ups": - finalResult = Map.of(underscoresToCamelCase(fieldName), new Gson().toJson(fieldValue)); + finalResult = new HashMap<>(Map.of(underscoresToCamelCase(fieldName), new Gson().toJson(fieldValue))); // List> followUps = convertObjectListToMapList(fieldValue); // finalResult = Map.of(underscoresToCamelCase(fieldName), followUps); break; @@ -130,7 +130,7 @@ static Map convertToMap(String fieldName, Object fieldValue) { finalResult = transformedMap; break; default: - finalResult = Map.of(underscoresToCamelCase(fieldName), fieldValue); + finalResult = new HashMap<>(Map.of(underscoresToCamelCase(fieldName), fieldValue)); break; } return finalResult; @@ -141,7 +141,7 @@ public static List> convertObjectListToMapList(Object fieldV } private static Map dynamicFieldsSpecialCase(Object fieldValue) { - Map dynamicMap = Map.of(); + Map dynamicMap = new HashMap<>(); if (isJsonInString(fieldValue)) { String strValue = (String) fieldValue; ObjectMapper objectMapper = new ObjectMapper(); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index b9a2cf3a6..7a2451f38 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -1,5 +1,6 @@ package org.broadinstitute.dsm.model.elastic.export; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -79,7 +80,9 @@ private Map processData(ESDsm esDsm) { processor.setCollector(generator); List> processedData = processor.process(); if (!processedData.isEmpty()) { - dataToReturn = Map.of(MappingGenerator.DSM_OBJECT, Map.of(propertyInfo.getPropertyName(), processedData)); + dataToReturn = new HashMap<>(Map.of(MappingGenerator.DSM_OBJECT, + new HashMap<>(Map.of(propertyInfo.getPropertyName(), + processedData)))); } } logger.info("Returning processed ES participant data"); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java index 6f0cef521..b2a75063a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java @@ -1,6 +1,7 @@ package org.broadinstitute.dsm.model.elastic.export.generate; import java.util.Collection; +import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -61,10 +62,18 @@ protected DBElement getDBElement() { return Util.getDBElement(getNameValue().getName()); } - protected PropertyInfo getOuterPropertyByAlias() { + private PropertyInfo getOuterPropertyByAlias() { return Util.TABLE_ALIAS_MAPPINGS.get(getDBElement().getTableAlias()); } + public String getPropertyName() { + return getOuterPropertyByAlias().getPropertyName(); + } + + public String getFieldName() { + return Util.underscoresToCamelCase(getDBElement().getColumnName()); + } + @Override public Object collect() { Object sourceToUpsert; @@ -93,7 +102,7 @@ protected Object parseSingleElement() { @Override public Map merge(Map base, Map toMerge) { - return Map.of(); + return new HashMap<>(); } public static class PropertyInfo { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java index 1918acbfe..0fca6b498 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java @@ -16,10 +16,10 @@ public CollectionMappingGenerator() {} @Override protected Map getElement(Object type) { - return Map.of( - Util.ID, Map.of(TYPE, TYPE_KEYWORD), + return new HashMap<>(Map.of( + Util.ID, new HashMap<>(Map.of(TYPE, TYPE_KEYWORD)), Util.underscoresToCamelCase(getDBElement().getColumnName()), type - ); + )); } @Override @@ -27,5 +27,8 @@ public Map construct() { return new HashMap<>(Map.of(TYPE, NESTED, PROPERTIES, collect())); } - + @Override + public Map merge(Map base, Map toMerge) { + return super.merge(base, toMerge); + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java index 08fe4d48a..ebe1efc81 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java @@ -34,7 +34,7 @@ public Object construct() { @Override protected Object getElement(Object element) { - return List.of(Map.of(Util.underscoresToCamelCase(getDBElement().getColumnName()), element, - Util.ID, generatorPayload.getRecordId())); + return List.of(new HashMap<>(Map.of(Util.underscoresToCamelCase(getDBElement().getColumnName()), element, + Util.ID, generatorPayload.getRecordId()))); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java index ca7e3460f..b5c6b3c70 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java @@ -29,11 +29,11 @@ public Map generate() { } public Map getCompleteMap(Object propertyMap) { - String propertyName = getOuterPropertyByAlias().getPropertyName(); - Map objectLevel = Map.of(propertyName, propertyMap); - Map dsmLevelProperties = Map.of(PROPERTIES, objectLevel); - Map> dsmLevel = Map.of(DSM_OBJECT, dsmLevelProperties); - return Map.of(PROPERTIES, dsmLevel); + String propertyName = getPropertyName(); + Map objectLevel = new HashMap<>(Map.of(propertyName, propertyMap)); + Map dsmLevelProperties = new HashMap<>(Map.of(PROPERTIES, objectLevel)); + Map> dsmLevel = new HashMap<>(Map.of(DSM_OBJECT, dsmLevelProperties)); + return new HashMap<>(Map.of(PROPERTIES, dsmLevel)); } @@ -45,7 +45,9 @@ protected Map parseJson() { Object eachType = parser.parse(String.valueOf(entry.getValue())); resultMap.put(Util.underscoresToCamelCase(entry.getKey()), eachType); } - return Map.of(Util.underscoresToCamelCase(getDBElement().getColumnName()), Map.of(PROPERTIES, resultMap)); + return new HashMap<>(Map.of(Util.underscoresToCamelCase(getDBElement().getColumnName()), + new HashMap<>(Map.of(PROPERTIES, + resultMap)))); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/NullObjectGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/NullObjectGenerator.java index b639d2a3b..6457a9d3f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/NullObjectGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/NullObjectGenerator.java @@ -1,5 +1,6 @@ package org.broadinstitute.dsm.model.elastic.export.generate; +import java.util.HashMap; import java.util.Map; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; @@ -7,7 +8,7 @@ public class NullObjectGenerator implements Generator { @Override public Map generate() { - return Map.of(); + return new HashMap<>(); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java index cab8c5b9d..030552678 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java @@ -16,7 +16,7 @@ public SingleMappingGenerator() {} @Override protected Map getElement(Object type) { - return Map.of(Util.underscoresToCamelCase(getDBElement().getColumnName()), type); + return new HashMap<>(Map.of(Util.underscoresToCamelCase(getDBElement().getColumnName()), type)); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleSourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleSourceGenerator.java index 735b9ffc5..4e5740a88 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleSourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleSourceGenerator.java @@ -1,5 +1,6 @@ package org.broadinstitute.dsm.model.elastic.export.generate; +import java.util.HashMap; import java.util.Map; import org.broadinstitute.dsm.model.elastic.Util; @@ -28,7 +29,7 @@ public Object construct() { @Override protected Map getElement(Object element) { logger.info("Constructing single field with value"); - return Map.of(Util.underscoresToCamelCase(getDBElement().getColumnName()), element); + return new HashMap<>(Map.of(Util.underscoresToCamelCase(getDBElement().getColumnName()), element)); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java index 7b7a5bfb3..9e285d69d 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java @@ -25,11 +25,11 @@ public SourceGenerator() { public Map generate() { Object dataToExport = collect(); logger.info("Generating final source"); - return Map.of(DSM_OBJECT, buildPropertyLevelWithData(dataToExport)); + return new HashMap<>(Map.of(DSM_OBJECT, buildPropertyLevelWithData(dataToExport))); } private Map buildPropertyLevelWithData(Object dataToExport) { - return Map.of(getOuterPropertyByAlias().getPropertyName(), dataToExport); + return new HashMap<>(Map.of(getPropertyName(), dataToExport)); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java index e4af19882..6a6472127 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java @@ -4,6 +4,7 @@ import com.google.gson.Gson; import org.broadinstitute.dsm.model.elastic.export.generate.MappingGenerator; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -14,11 +15,15 @@ public class TypeParser extends BaseParser { private static final String FIELDS = "fields"; private static final String KEYWORD = "keyword"; private static final String TYPE = "type"; - public static final Map TEXT_KEYWORD_MAPPING = Map.of(TYPE, TEXT, FIELDS, Map.of(KEYWORD, Map.of(TYPE, KEYWORD))); + public static final Map TEXT_KEYWORD_MAPPING = new HashMap<>( + new HashMap<>( + Map.of(TYPE, TEXT, + FIELDS, new HashMap<>(Map.of(KEYWORD, new HashMap<>(Map.of(TYPE, KEYWORD)) + ))))); private static final String BOOLEAN = "boolean"; - public static final Map BOOLEAN_MAPPING = Map.of(MappingGenerator.TYPE, BOOLEAN); + public static final Map BOOLEAN_MAPPING = new HashMap<>(Map.of(MappingGenerator.TYPE, BOOLEAN)); private static final String DATE = "date"; - public static final Map DATE_MAPPING = Map.of(MappingGenerator.TYPE, DATE); + public static final Map DATE_MAPPING = new HashMap<>(Map.of(MappingGenerator.TYPE, DATE)); @Override protected Object forNumeric(String value) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java index 358ca4dec..4b6c0c9dc 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java @@ -15,12 +15,13 @@ import org.broadinstitute.dsm.model.elastic.export.generate.CollectionMappingGenerator; import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; import org.broadinstitute.dsm.model.elastic.export.generate.MappingGenerator; +import org.broadinstitute.dsm.model.elastic.export.generate.Merger; import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; import org.broadinstitute.dsm.statics.ESObjectConstants; -public abstract class BaseCollectionMigrator extends BaseMigrator { +public abstract class BaseCollectionMigrator extends BaseMigrator implements Merger { - private final MappingGenerator collectionMappingGenerator = new CollectionMappingGenerator(); + private final BaseGenerator collectionMappingGenerator = new CollectionMappingGenerator(); protected List> transformedList; protected Set primaryKeys; private Map endResult = new HashMap<>(); @@ -32,7 +33,7 @@ public BaseCollectionMigrator(String index, String realm, String object) { @Override public Map generate() { - return Map.of(ESObjectConstants.DSM, Map.of(object, transformedList)); + return new HashMap<>(Map.of(ESObjectConstants.DSM, new HashMap<>(Map.of(object, transformedList)))); } @Override @@ -57,7 +58,7 @@ protected void transformObject(Object object) { NameValue nameValue = new NameValue(fieldNameWithAlias, fieldValue); GeneratorPayload generatorPayload = new GeneratorPayload(nameValue); collectionMappingGenerator.setPayload(generatorPayload); - endResult = collectionMappingGenerator.merge(endResult, (Map) collectionMappingGenerator.construct()); + endResult = merge(endResult, collectionMappingGenerator.generate()); } catch (IllegalAccessException e) { throw new RuntimeException(e); } @@ -67,6 +68,41 @@ protected void transformObject(Object object) { setPrimaryId(); } + @Override + public Map merge(Map base, Map toMerge) { + if (base.isEmpty()) { + return toMerge; + } + getFieldLevel(base).putAll(getFieldLevel(toMerge)); + return base; + } + + private Map getFieldLevel(Map base) { + Map result = new HashMap<>(); + Map propertyMap = getPropertyLevel(base); + Map innerProperty = (Map)propertyMap.get("properties"); + if (innerProperty == null) result = propertyMap; + Map fieldProperty = (Map) innerProperty.get(collectionMappingGenerator.getFieldName()); + if (fieldProperty == null) result = innerProperty; + if (fieldProperty != null) { + Map fieldProperties = (Map)fieldProperty.get("properties"); + if (fieldProperties != null) { + result = fieldProperties; + } else { + result = innerProperty; + } + } + return result; + } + + private Map getPropertyLevel(Map base) { + Map topLevelProperties = (Map) base.get("properties"); + Map dsmProperties = (Map) topLevelProperties.get("dsm"); + Map properties = (Map) dsmProperties.get("properties"); + Map propertyMap = (Map)properties.get(collectionMappingGenerator.getPropertyName()); + return propertyMap; + } + private void collectPrimaryKeys(Object obj) { Class clazz = obj.getClass(); extractAndCollectPrimaryKey(clazz); @@ -152,7 +188,7 @@ private void putPrimaryId(Map map, String outerKey) { protected void exportMap() { ElasticMappingExportAdapter elasticMappingExportAdapter = new ElasticMappingExportAdapter(); elasticMappingExportAdapter.setRequestPayload(new RequestPayload(index)); - elasticMappingExportAdapter.setSource(collectionMappingGenerator.getCompleteMap(endResult)); + elasticMappingExportAdapter.setSource(endResult); elasticMappingExportAdapter.export(); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseSingleMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseSingleMigrator.java index 27c18b287..e4428221b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseSingleMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseSingleMigrator.java @@ -1,5 +1,6 @@ package org.broadinstitute.dsm.model.elastic.migration; +import java.util.HashMap; import java.util.Map; import org.broadinstitute.dsm.model.elastic.Util; @@ -15,7 +16,7 @@ public BaseSingleMigrator(String index, String realm, String object) { @Override public Map generate() { - return Map.of(ESObjectConstants.DSM, Map.of(object, transformedObject)); + return new HashMap<>(Map.of(ESObjectConstants.DSM, new HashMap<>(Map.of(object, transformedObject)))); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java index b25bb60d0..b6852895a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java @@ -5,6 +5,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -285,7 +286,8 @@ public ElasticSearchParticipantDto getParticipantById(String esParticipantsIndex logger.info("Collecting ES data"); try { searchResponse = ElasticSearchUtil.getClientInstance().search(searchRequest, RequestOptions.DEFAULT); - sourceAsMap = searchResponse.getHits().getHits().length > 0 ? searchResponse.getHits().getHits()[0].getSourceAsMap() : Map.of(); + sourceAsMap = searchResponse.getHits().getHits().length > 0 ? + searchResponse.getHits().getHits()[0].getSourceAsMap() : new HashMap<>(); } catch (Exception e) { throw new RuntimeException("Couldn't get participant from ES for instance " + esParticipantsIndex + " by id: " + participantId, e); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java index 0eaca236a..44ac28610 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java @@ -85,7 +85,7 @@ public void generateMapping() { ); MappingGenerator generator = TestCollectionMappingGenerator.of(generatorPayload); Map resultMap = generator.generate(); - Map dsmLevelProperty = Map.of(generator.getOuterPropertyByAlias().getPropertyName(), Map.of( + Map dsmLevelProperty = Map.of(generator.getPropertyName(), Map.of( MappingGenerator.TYPE, MappingGenerator.NESTED, PROPERTIES, Map.of(Util.underscoresToCamelCase(TestPatchUtil.MEDICAL_RECORD_COLUMN), Map.of(MappingGenerator.TYPE, "date"), Util.ID, Map.of(MappingGenerator.TYPE, MappingGenerator.TYPE_KEYWORD) @@ -112,14 +112,14 @@ public void merge() { .get(PROPERTIES)) .get(MappingGenerator.DSM_OBJECT)) .get(PROPERTIES)) - .get(mappingGenerator.getOuterPropertyByAlias().getPropertyName())) + .get(mappingGenerator.getPropertyName())) .get(PROPERTIES)) .get(Util.underscoresToCamelCase("DDP_INSTANCE1")); Object value2 = ((Map) ((Map) ((Map) ((Map) ((Map) base .get(PROPERTIES)) .get(MappingGenerator.DSM_OBJECT)) .get(PROPERTIES)) - .get(mappingGenerator.getOuterPropertyByAlias().getPropertyName())) + .get(mappingGenerator.getPropertyName())) .get(PROPERTIES)) .get(Util.underscoresToCamelCase("DDP_INSTANCE")); Assert.assertFalse(Objects.isNull(value)); From 778e09b7bcb7a745b5a5640168b4392b8cc232d2 Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 29 Nov 2021 18:03:47 +0400 Subject: [PATCH 164/405] DDP-7121 | fix bug --- .../model/elastic/migration/BaseCollectionMigrator.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java index 4b6c0c9dc..de52b89d6 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java @@ -19,6 +19,8 @@ import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; import org.broadinstitute.dsm.statics.ESObjectConstants; +import static org.broadinstitute.dsm.util.ElasticSearchUtil.PROPERTIES; + public abstract class BaseCollectionMigrator extends BaseMigrator implements Merger { private final BaseGenerator collectionMappingGenerator = new CollectionMappingGenerator(); @@ -83,11 +85,14 @@ private Map getFieldLevel(Map base) { Map innerProperty = (Map)propertyMap.get("properties"); if (innerProperty == null) result = propertyMap; Map fieldProperty = (Map) innerProperty.get(collectionMappingGenerator.getFieldName()); - if (fieldProperty == null) result = innerProperty; + if (fieldProperty == null) { + + result = innerProperty; + } if (fieldProperty != null) { Map fieldProperties = (Map)fieldProperty.get("properties"); if (fieldProperties != null) { - result = fieldProperties; + result = fieldProperty; } else { result = innerProperty; } From 5361d3bc7f919c0773e2fefe6fcb0f824af0e2cf Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 30 Nov 2021 10:37:24 +0400 Subject: [PATCH 165/405] DDP-7121 | add dynamic fields mapping migrator --- .../migration/BaseCollectionMigrator.java | 20 ++++++++++- .../DynamicFieldsMappingMigrator.java | 34 +++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java index de52b89d6..8987dca79 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java @@ -75,7 +75,25 @@ public Map merge(Map base, Map t if (base.isEmpty()) { return toMerge; } - getFieldLevel(base).putAll(getFieldLevel(toMerge)); + String fieldName = collectionMappingGenerator.getFieldName(); + Map basePropertyLevel = getPropertyLevel(base); + Map toMergePropertyLevel = getPropertyLevel(toMerge); + Map baseProperties = (Map) basePropertyLevel.get("properties"); + Map toMergeProperties = (Map) toMergePropertyLevel.get("properties"); + Map baseField = (Map) baseProperties.get(fieldName); + Map toMergeField = (Map) toMergeProperties.get(fieldName); + if (baseField == null) { + base.put(fieldName, getFieldLevel(toMerge)); + } + Map baseFieldProperties = (Map) baseField.get("properties"); + Map toMergeFieldProperties = (Map) toMergeField.get("properties"); + if (baseFieldProperties == null && toMergeFieldProperties != null) { + baseField.put("properties", getFieldLevel(toMerge)); + } else if (toMergeFieldProperties != null) { + baseFieldProperties.putAll(getFieldLevel(toMerge)); + } else { + getFieldLevel(base).putAll(getFieldLevel(toMerge)); + } return base; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java new file mode 100644 index 000000000..39e84bcb5 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java @@ -0,0 +1,34 @@ +package org.broadinstitute.dsm.model.elastic.migration; + +import java.util.Map; + +import org.broadinstitute.dsm.db.dao.settings.FieldSettingsDao; +import org.broadinstitute.dsm.model.elastic.export.ElasticMappingExportAdapter; +import org.broadinstitute.dsm.model.elastic.export.Exportable; +import org.broadinstitute.dsm.model.elastic.export.RequestPayload; + +public class DynamicFieldsMappingMigrator implements Exportable { + + private ElasticMappingExportAdapter elasticMappingExportAdapter = new ElasticMappingExportAdapter(); + + public DynamicFieldsMappingMigrator(String index, String study) { + + } + + @Override + public void export() { + FieldSettingsDao fieldSettingsDao = FieldSettingsDao.of(); + fieldSettingsDao.getFieldSettingsByInstanceId() + elasticMappingExportAdapter.export(); + } + + @Override + public void setSource(Map source) { + + } + + @Override + public void setRequestPayload(RequestPayload requestPayload) { + + } +} From 6dd0143caed9671efe46f0646d27ef8f0473f32e Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 30 Nov 2021 11:08:34 +0400 Subject: [PATCH 166/405] DDP-7121 | start creating type parser for dynamic fields --- .../dsm/db/dao/settings/FieldSettingsDao.java | 57 +++++++++++++++++++ .../DynamicFieldsMappingMigrator.java | 45 ++++++++++++++- 2 files changed, 100 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/dao/settings/FieldSettingsDao.java b/src/main/java/org/broadinstitute/dsm/db/dao/settings/FieldSettingsDao.java index fe6961c53..106ef9a00 100644 --- a/src/main/java/org/broadinstitute/dsm/db/dao/settings/FieldSettingsDao.java +++ b/src/main/java/org/broadinstitute/dsm/db/dao/settings/FieldSettingsDao.java @@ -12,6 +12,7 @@ import java.util.stream.Collectors; import org.broadinstitute.ddp.db.SimpleResult; +import org.broadinstitute.dsm.db.FieldSettings; import org.broadinstitute.dsm.db.dao.Dao; import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; @@ -35,6 +36,22 @@ public class FieldSettingsDao implements Dao { "changed_by" + " FROM field_settings WHERE ddp_instance_id = ? and (display_type = 'OPTIONS' or display_type = 'RADIO') "; + private static final String SQL_FIELD_SETTINGS_BY_STUDY_NAME = "SELECT " + + "field_settings_id," + + "ddp_instance_id," + + "field_type," + + "column_name," + + "column_display," + + "display_type," + + "possible_values," + + "actions," + + "readonly," + + "order_number," + + "deleted," + + "last_changed," + + "changed_by" + + " FROM field_settings WHERE ddp_instance_id = (SELECT ddp_instance_id FROM ddp_instance WHERE instance_name = ?)"; + private static final String GET_FIELD_SETTINGS = "SELECT " + "field_settings_id," + "ddp_instance_id," + @@ -194,6 +211,46 @@ public List getOptionAndRadioFieldSettingsByInstanceId(int ins return fieldSettingsByOptions; } + public List getFieldSettingsByStudyName(String studyName) { + + List fieldSettingsByOptions = new ArrayList<>(); + SimpleResult results = inTransaction((conn) -> { + SimpleResult execResult = new SimpleResult(); + try (PreparedStatement stmt = conn.prepareStatement(SQL_FIELD_SETTINGS_BY_STUDY_NAME)) { + stmt.setString(1, studyName); + try(ResultSet rs = stmt.executeQuery()) { + while (rs.next()) { + fieldSettingsByOptions.add( + new FieldSettingsDto.Builder(rs.getInt(DDP_INSTANCE_ID)) + .withFieldSettingsId(rs.getInt(FIELD_SETTINGS_ID)) + .withFieldType(rs.getString(FIELD_TYPE)) + .withColumnName(rs.getString(COLUMN_NAME)) + .withColumnDisplay(rs.getString(COLUMN_DISPLAY)) + .withDisplayType(rs.getString(DISPLAY_TYPE)) + .withPossibleValues(rs.getString(POSSIBLE_VALUES)) + .withActions(rs.getString(ACTIONS)) + .withReadOnly(rs.getBoolean(READONLY)) + .withOrderNumber(rs.getInt(ORDER_NUMBER)) + .withDeleted(rs.getBoolean(DELETED)) + .withLastChanged(rs.getLong(LAST_CHANGED)) + .withChangedBy(rs.getString(CHANGED_BY)) + .build() + ); + } + } + } + catch (SQLException ex) { + execResult.resultException = ex; + } + return execResult; + }); + if (results.resultException != null) { + throw new RuntimeException("Error getting fieldSettings for study: " + + studyName, results.resultException); + } + return fieldSettingsByOptions; + } + public List getFieldSettingsByInstanceId(int instanceId) { List fieldSettingsByOptions = new ArrayList<>(); SimpleResult results = inTransaction((conn) -> { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java index 39e84bcb5..f5aee545e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java @@ -1,24 +1,53 @@ package org.broadinstitute.dsm.model.elastic.migration; +import java.util.HashMap; +import java.util.List; import java.util.Map; import org.broadinstitute.dsm.db.dao.settings.FieldSettingsDao; +import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; +import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.ElasticMappingExportAdapter; import org.broadinstitute.dsm.model.elastic.export.Exportable; import org.broadinstitute.dsm.model.elastic.export.RequestPayload; +import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; +import org.broadinstitute.dsm.model.elastic.export.parse.BaseParser; +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; +import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; + +import static org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator.DSM_OBJECT; +import static org.broadinstitute.dsm.util.ElasticSearchUtil.PROPERTIES; public class DynamicFieldsMappingMigrator implements Exportable { + private final String index; + private final String study; + private ElasticMappingExportAdapter elasticMappingExportAdapter = new ElasticMappingExportAdapter(); public DynamicFieldsMappingMigrator(String index, String study) { - + this.index = index; + this.study = study; } @Override public void export() { FieldSettingsDao fieldSettingsDao = FieldSettingsDao.of(); - fieldSettingsDao.getFieldSettingsByInstanceId() + List fieldSettingsByStudyName = fieldSettingsDao.getFieldSettingsByStudyName(study); + for (FieldSettingsDto fieldSettingsDto : fieldSettingsByStudyName) { + String fieldType = fieldSettingsDto.getFieldType(); + // oD, m,t + BaseGenerator.PropertyInfo propertyInfo = Util.TABLE_ALIAS_MAPPINGS.get(fieldType); + if (propertyInfo != null) { + String propertyName = propertyInfo.getPropertyName(); + new HashMap<>(Map.of(propertyName, new HashMap<>(Map.of(PROPERTIES,"")))); + Map dsmLevelProperties = new HashMap<>(Map.of(PROPERTIES, objectLevel)); + Map> dsmLevel = new HashMap<>(Map.of(DSM_OBJECT, dsmLevelProperties)); + Map finalMap = new HashMap<>(Map.of(PROPERTIES, dsmLevel)); + } + + + } elasticMappingExportAdapter.export(); } @@ -31,4 +60,16 @@ public void setSource(Map source) { public void setRequestPayload(RequestPayload requestPayload) { } + +} + +class DynamicFieldsTypeParser extends TypeParser { + + @Override + public Object parse(String value) { + + switch (value) { + + } + } } From b6a9aa8bfab57745007c5c405c86584abd4b8b5a Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 30 Nov 2021 11:39:00 +0400 Subject: [PATCH 167/405] DDP-7121 | continue implementing of dynamic fields mapping migrator --- .../elastic/export/parse/BaseParser.java | 3 +- .../DynamicFieldsMappingMigrator.java | 28 ++++++++++++++----- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java index 6b70bab3d..bb6300d12 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java @@ -67,7 +67,8 @@ boolean isDate(String value) { } boolean isBoolean(String value) { - return value.equalsIgnoreCase("true") || value.equalsIgnoreCase("false"); + return value.equalsIgnoreCase("true") || + value.equalsIgnoreCase("false"); } boolean isCollection(String value) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java index f5aee545e..f3103171f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java @@ -34,20 +34,31 @@ public DynamicFieldsMappingMigrator(String index, String study) { public void export() { FieldSettingsDao fieldSettingsDao = FieldSettingsDao.of(); List fieldSettingsByStudyName = fieldSettingsDao.getFieldSettingsByStudyName(study); + Parser parser = new DynamicFieldsTypeParser(); + Map propertyMap = new HashMap<>(); + Map additionalValuesJson = new HashMap<>(Map.of("additionalValuesJson", new HashMap<>())); +// Map.of(propertyName, new HashMap<>(Map.of(PROPERTIES, ""))) for (FieldSettingsDto fieldSettingsDto : fieldSettingsByStudyName) { String fieldType = fieldSettingsDto.getFieldType(); // oD, m,t BaseGenerator.PropertyInfo propertyInfo = Util.TABLE_ALIAS_MAPPINGS.get(fieldType); if (propertyInfo != null) { String propertyName = propertyInfo.getPropertyName(); - new HashMap<>(Map.of(propertyName, new HashMap<>(Map.of(PROPERTIES,"")))); - Map dsmLevelProperties = new HashMap<>(Map.of(PROPERTIES, objectLevel)); - Map> dsmLevel = new HashMap<>(Map.of(DSM_OBJECT, dsmLevelProperties)); - Map finalMap = new HashMap<>(Map.of(PROPERTIES, dsmLevel)); + Object typeMap = parser.parse(fieldSettingsDto.getDisplayType()); + Map medicalRecord = (Map) propertyMap.get(propertyName); + medicalRecord + propertyMap.merge(propertyName, typeMap, (prev, curr) -> { + Map valuesJson = (Map) ((Map) prev).get("additionalValuesJson"); + valuesJson.putAll((Map)curr); + return prev; + }); } } + Map dsmLevelProperties = new HashMap<>(Map.of(PROPERTIES, propertyMap)); + Map> dsmLevel = new HashMap<>(Map.of(DSM_OBJECT, dsmLevelProperties)); + Map finalMap = new HashMap<>(Map.of(PROPERTIES, dsmLevel)); elasticMappingExportAdapter.export(); } @@ -67,9 +78,12 @@ class DynamicFieldsTypeParser extends TypeParser { @Override public Object parse(String value) { - - switch (value) { - + Object parsedValue; + if ("DATE".equals(value)) { + parsedValue = forDate(value); + } else { + parsedValue = forString(value); } + return parsedValue; } } From 1c828c3239aca11afd3fffe53faa3f5ef4f1e168 Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 30 Nov 2021 12:18:49 +0400 Subject: [PATCH 168/405] DDP-7121 | generate mapping for dynamic fields --- .../DynamicFieldsMappingMigrator.java | 34 +++++++++++++------ .../DynamicFieldsMappingMigratorTest.java | 21 ++++++++++++ 2 files changed, 45 insertions(+), 10 deletions(-) create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java index f3103171f..1325d7a6c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java @@ -36,25 +36,39 @@ public void export() { List fieldSettingsByStudyName = fieldSettingsDao.getFieldSettingsByStudyName(study); Parser parser = new DynamicFieldsTypeParser(); Map propertyMap = new HashMap<>(); - Map additionalValuesJson = new HashMap<>(Map.of("additionalValuesJson", new HashMap<>())); // Map.of(propertyName, new HashMap<>(Map.of(PROPERTIES, ""))) for (FieldSettingsDto fieldSettingsDto : fieldSettingsByStudyName) { String fieldType = fieldSettingsDto.getFieldType(); // oD, m,t BaseGenerator.PropertyInfo propertyInfo = Util.TABLE_ALIAS_MAPPINGS.get(fieldType); if (propertyInfo != null) { + String columnName = Util.underscoresToCamelCase(fieldSettingsDto.getColumnName()); String propertyName = propertyInfo.getPropertyName(); Object typeMap = parser.parse(fieldSettingsDto.getDisplayType()); - Map medicalRecord = (Map) propertyMap.get(propertyName); - medicalRecord - propertyMap.merge(propertyName, typeMap, (prev, curr) -> { - Map valuesJson = (Map) ((Map) prev).get("additionalValuesJson"); - valuesJson.putAll((Map)curr); - return prev; - }); + if (!(propertyMap.containsKey(propertyName))) { + Map additionalValuesJson = new HashMap<>(Map.of("additionalValuesJson", new HashMap<>(Map.of(PROPERTIES, new HashMap<>(Map.of(columnName, typeMap)))))); + propertyMap.put(propertyName, additionalValuesJson); + } else { + Map map = (Map) propertyMap.get(propertyName); + Map json = (Map) map.get("additionalValuesJson"); + Map properties = (Map) json.get(PROPERTIES); + properties.putIfAbsent(columnName, typeMap); + } + } else { + String columnName = Util.underscoresToCamelCase(fieldSettingsDto.getColumnName()); + String propertyName = "participantData"; + Object typeMap = parser.parse(fieldSettingsDto.getDisplayType()); + if (!(propertyMap.containsKey(propertyName))) { + Map additionalValuesJson = new HashMap<>(Map.of("data", new HashMap<>())); + additionalValuesJson.put(PROPERTIES, new HashMap<>(Map.of(columnName, typeMap))); + propertyMap.put(propertyName, additionalValuesJson); + } else { + Map map = (Map) propertyMap.get(propertyName); + Map json = (Map) map.get("data"); + Map properties = (Map) json.get(PROPERTIES); + properties.putIfAbsent(columnName, typeMap); + } } - - } Map dsmLevelProperties = new HashMap<>(Map.of(PROPERTIES, propertyMap)); Map> dsmLevel = new HashMap<>(Map.of(DSM_OBJECT, dsmLevelProperties)); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java new file mode 100644 index 000000000..099d2e3b2 --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java @@ -0,0 +1,21 @@ +package org.broadinstitute.dsm.model.elastic.migration; + +import junit.framework.TestCase; +import org.broadinstitute.dsm.TestHelper; +import org.broadinstitute.dsm.model.elastic.export.Exportable; +import org.junit.BeforeClass; +import org.junit.Test; + +public class DynamicFieldsMappingMigratorTest { + + @BeforeClass + public static void setUp() { + TestHelper.setupDB(); + } + + @Test + public void testExport() { + Exportable exportable = new DynamicFieldsMappingMigrator("", "angio"); + exportable.export(); + } +} \ No newline at end of file From 46893a6b14e6f4dc2dcb4e7961e21f8633de1d6c Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 30 Nov 2021 17:05:19 +0400 Subject: [PATCH 169/405] DDP-7121 | refactor DynamicFieldsMappingGenerator.java --- .../DynamicFieldsMappingMigrator.java | 69 ++++++++++--------- .../DynamicFieldsMappingMigratorTest.java | 2 +- 2 files changed, 37 insertions(+), 34 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java index 1325d7a6c..d6a747baa 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java @@ -22,58 +22,61 @@ public class DynamicFieldsMappingMigrator implements Exportable { private final String index; private final String study; + public Parser parser; + public Map propertyMap; private ElasticMappingExportAdapter elasticMappingExportAdapter = new ElasticMappingExportAdapter(); public DynamicFieldsMappingMigrator(String index, String study) { this.index = index; this.study = study; + this.parser = new DynamicFieldsTypeParser(); + this.propertyMap = new HashMap<>(); } @Override public void export() { FieldSettingsDao fieldSettingsDao = FieldSettingsDao.of(); List fieldSettingsByStudyName = fieldSettingsDao.getFieldSettingsByStudyName(study); - Parser parser = new DynamicFieldsTypeParser(); - Map propertyMap = new HashMap<>(); -// Map.of(propertyName, new HashMap<>(Map.of(PROPERTIES, ""))) for (FieldSettingsDto fieldSettingsDto : fieldSettingsByStudyName) { String fieldType = fieldSettingsDto.getFieldType(); - // oD, m,t BaseGenerator.PropertyInfo propertyInfo = Util.TABLE_ALIAS_MAPPINGS.get(fieldType); - if (propertyInfo != null) { - String columnName = Util.underscoresToCamelCase(fieldSettingsDto.getColumnName()); - String propertyName = propertyInfo.getPropertyName(); - Object typeMap = parser.parse(fieldSettingsDto.getDisplayType()); - if (!(propertyMap.containsKey(propertyName))) { - Map additionalValuesJson = new HashMap<>(Map.of("additionalValuesJson", new HashMap<>(Map.of(PROPERTIES, new HashMap<>(Map.of(columnName, typeMap)))))); - propertyMap.put(propertyName, additionalValuesJson); - } else { - Map map = (Map) propertyMap.get(propertyName); - Map json = (Map) map.get("additionalValuesJson"); - Map properties = (Map) json.get(PROPERTIES); - properties.putIfAbsent(columnName, typeMap); - } - } else { - String columnName = Util.underscoresToCamelCase(fieldSettingsDto.getColumnName()); - String propertyName = "participantData"; - Object typeMap = parser.parse(fieldSettingsDto.getDisplayType()); - if (!(propertyMap.containsKey(propertyName))) { - Map additionalValuesJson = new HashMap<>(Map.of("data", new HashMap<>())); - additionalValuesJson.put(PROPERTIES, new HashMap<>(Map.of(columnName, typeMap))); - propertyMap.put(propertyName, additionalValuesJson); - } else { - Map map = (Map) propertyMap.get(propertyName); - Map json = (Map) map.get("data"); - Map properties = (Map) json.get(PROPERTIES); - properties.putIfAbsent(columnName, typeMap); - } - } + if (propertyInfo != null) + buildMapping(fieldSettingsDto, propertyInfo, "additionalValuesJson"); + else + buildMapping(fieldSettingsDto, new BaseGenerator.PropertyInfo("participantData", true), "data"); } + elasticMappingExportAdapter.setRequestPayload(new RequestPayload(index)); + elasticMappingExportAdapter.setSource(buildFinalMapping()); + elasticMappingExportAdapter.export(); + } + + private Map buildFinalMapping() { Map dsmLevelProperties = new HashMap<>(Map.of(PROPERTIES, propertyMap)); Map> dsmLevel = new HashMap<>(Map.of(DSM_OBJECT, dsmLevelProperties)); Map finalMap = new HashMap<>(Map.of(PROPERTIES, dsmLevel)); - elasticMappingExportAdapter.export(); + return finalMap; + } + + private void buildMapping(FieldSettingsDto fieldSettingsDto, BaseGenerator.PropertyInfo propertyInfo, String dynamicFieldsWrapperName) { + String columnName = Util.underscoresToCamelCase(fieldSettingsDto.getColumnName()); + String propertyName = propertyInfo.getPropertyName(); + Object typeMap = parser.parse(fieldSettingsDto.getDisplayType()); + if (!(propertyMap.containsKey(propertyName))) { + Map additionalValuesJson = new HashMap<>(Map.of(dynamicFieldsWrapperName, new HashMap<>(Map.of(PROPERTIES, new HashMap<>(Map.of(columnName, typeMap)))))); + Map wrapperMap = new HashMap<>(); + if (propertyInfo.isCollection()) { + wrapperMap.put("type", "nested"); + } + wrapperMap.put(PROPERTIES, additionalValuesJson); + propertyMap.put(propertyName, wrapperMap); + } else { + Map outerMap = (Map) propertyMap.get(propertyName); + Map outerProperties = (Map) outerMap.get(PROPERTIES); + Map dynamicFieldsJson = (Map) outerProperties.get(dynamicFieldsWrapperName); + Map innerProperties = (Map) dynamicFieldsJson.get(PROPERTIES); + innerProperties.putIfAbsent(columnName, typeMap); + } } @Override diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java index 099d2e3b2..81bf5136e 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java @@ -15,7 +15,7 @@ public static void setUp() { @Test public void testExport() { - Exportable exportable = new DynamicFieldsMappingMigrator("", "angio"); + Exportable exportable = new DynamicFieldsMappingMigrator("participants_structured.cmi.angio", "angio"); exportable.export(); } } \ No newline at end of file From 9a9f878425e32eec4b5caca822db96ff85e7ee34 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 30 Nov 2021 17:40:45 +0400 Subject: [PATCH 170/405] DDP-7121 | clean up of migration classes --- .../migration/BaseCollectionMigrator.java | 66 +------------------ .../model/elastic/migration/BaseMigrator.java | 3 - .../DynamicFieldsMappingMigrator.java | 19 +++--- .../migration/ParticipantDataMigrator.java | 43 ------------ .../dsm/pubsub/DSMtasksSubscription.java | 5 +- .../dsm/statics/ESObjectConstants.java | 1 + 6 files changed, 17 insertions(+), 120 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java index 8987dca79..5c7996718 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java @@ -21,7 +21,7 @@ import static org.broadinstitute.dsm.util.ElasticSearchUtil.PROPERTIES; -public abstract class BaseCollectionMigrator extends BaseMigrator implements Merger { +public abstract class BaseCollectionMigrator extends BaseMigrator { private final BaseGenerator collectionMappingGenerator = new CollectionMappingGenerator(); protected List> transformedList; @@ -70,62 +70,6 @@ protected void transformObject(Object object) { setPrimaryId(); } - @Override - public Map merge(Map base, Map toMerge) { - if (base.isEmpty()) { - return toMerge; - } - String fieldName = collectionMappingGenerator.getFieldName(); - Map basePropertyLevel = getPropertyLevel(base); - Map toMergePropertyLevel = getPropertyLevel(toMerge); - Map baseProperties = (Map) basePropertyLevel.get("properties"); - Map toMergeProperties = (Map) toMergePropertyLevel.get("properties"); - Map baseField = (Map) baseProperties.get(fieldName); - Map toMergeField = (Map) toMergeProperties.get(fieldName); - if (baseField == null) { - base.put(fieldName, getFieldLevel(toMerge)); - } - Map baseFieldProperties = (Map) baseField.get("properties"); - Map toMergeFieldProperties = (Map) toMergeField.get("properties"); - if (baseFieldProperties == null && toMergeFieldProperties != null) { - baseField.put("properties", getFieldLevel(toMerge)); - } else if (toMergeFieldProperties != null) { - baseFieldProperties.putAll(getFieldLevel(toMerge)); - } else { - getFieldLevel(base).putAll(getFieldLevel(toMerge)); - } - return base; - } - - private Map getFieldLevel(Map base) { - Map result = new HashMap<>(); - Map propertyMap = getPropertyLevel(base); - Map innerProperty = (Map)propertyMap.get("properties"); - if (innerProperty == null) result = propertyMap; - Map fieldProperty = (Map) innerProperty.get(collectionMappingGenerator.getFieldName()); - if (fieldProperty == null) { - - result = innerProperty; - } - if (fieldProperty != null) { - Map fieldProperties = (Map)fieldProperty.get("properties"); - if (fieldProperties != null) { - result = fieldProperty; - } else { - result = innerProperty; - } - } - return result; - } - - private Map getPropertyLevel(Map base) { - Map topLevelProperties = (Map) base.get("properties"); - Map dsmProperties = (Map) topLevelProperties.get("dsm"); - Map properties = (Map) dsmProperties.get("properties"); - Map propertyMap = (Map)properties.get(collectionMappingGenerator.getPropertyName()); - return propertyMap; - } - private void collectPrimaryKeys(Object obj) { Class clazz = obj.getClass(); extractAndCollectPrimaryKey(clazz); @@ -206,12 +150,4 @@ private Optional getPrimaryKey(Map map) { private void putPrimaryId(Map map, String outerKey) { map.put(Util.ID, map.get(outerKey)); } - - @Override - protected void exportMap() { - ElasticMappingExportAdapter elasticMappingExportAdapter = new ElasticMappingExportAdapter(); - elasticMappingExportAdapter.setRequestPayload(new RequestPayload(index)); - elasticMappingExportAdapter.setSource(endResult); - elasticMappingExportAdapter.export(); - } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseMigrator.java index 29faaa2ad..f6d08ed78 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseMigrator.java @@ -46,12 +46,9 @@ protected void fillBulkRequestWithTransformedMap(Map participant protected abstract Map getDataByRealm(); - protected void exportMap() {} - @Override public void export() { fillBulkRequestWithTransformedMap(getDataByRealm()); - exportMap(); bulkExportFacade.executeBulkUpsert(); logger.info("finished migrating data to ES."); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java index d6a747baa..06bf8eda5 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java @@ -1,5 +1,8 @@ package org.broadinstitute.dsm.model.elastic.migration; +import static org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator.DSM_OBJECT; +import static org.broadinstitute.dsm.util.ElasticSearchUtil.PROPERTIES; + import java.util.HashMap; import java.util.List; import java.util.Map; @@ -11,12 +14,11 @@ import org.broadinstitute.dsm.model.elastic.export.Exportable; import org.broadinstitute.dsm.model.elastic.export.RequestPayload; import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; -import org.broadinstitute.dsm.model.elastic.export.parse.BaseParser; +import org.broadinstitute.dsm.model.elastic.export.generate.MappingGenerator; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; - -import static org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator.DSM_OBJECT; -import static org.broadinstitute.dsm.util.ElasticSearchUtil.PROPERTIES; +import org.broadinstitute.dsm.statics.DBConstants; +import org.broadinstitute.dsm.statics.ESObjectConstants; public class DynamicFieldsMappingMigrator implements Exportable { @@ -25,13 +27,14 @@ public class DynamicFieldsMappingMigrator implements Exportable { public Parser parser; public Map propertyMap; - private ElasticMappingExportAdapter elasticMappingExportAdapter = new ElasticMappingExportAdapter(); + private ElasticMappingExportAdapter elasticMappingExportAdapter; public DynamicFieldsMappingMigrator(String index, String study) { this.index = index; this.study = study; this.parser = new DynamicFieldsTypeParser(); this.propertyMap = new HashMap<>(); + elasticMappingExportAdapter = new ElasticMappingExportAdapter(); } @Override @@ -42,9 +45,9 @@ public void export() { String fieldType = fieldSettingsDto.getFieldType(); BaseGenerator.PropertyInfo propertyInfo = Util.TABLE_ALIAS_MAPPINGS.get(fieldType); if (propertyInfo != null) - buildMapping(fieldSettingsDto, propertyInfo, "additionalValuesJson"); + buildMapping(fieldSettingsDto, propertyInfo, ESObjectConstants.ADDITIONAL_VALUES_JSON); else - buildMapping(fieldSettingsDto, new BaseGenerator.PropertyInfo("participantData", true), "data"); + buildMapping(fieldSettingsDto, new BaseGenerator.PropertyInfo(ESObjectConstants.PARTICIPANT_DATA, true), DBConstants.DATA); } elasticMappingExportAdapter.setRequestPayload(new RequestPayload(index)); elasticMappingExportAdapter.setSource(buildFinalMapping()); @@ -66,7 +69,7 @@ private void buildMapping(FieldSettingsDto fieldSettingsDto, BaseGenerator.Prope Map additionalValuesJson = new HashMap<>(Map.of(dynamicFieldsWrapperName, new HashMap<>(Map.of(PROPERTIES, new HashMap<>(Map.of(columnName, typeMap)))))); Map wrapperMap = new HashMap<>(); if (propertyInfo.isCollection()) { - wrapperMap.put("type", "nested"); + wrapperMap.put(MappingGenerator.TYPE, MappingGenerator.NESTED); } wrapperMap.put(PROPERTIES, additionalValuesJson); propertyMap.put(propertyName, wrapperMap); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java index 51363213f..b1bace846 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java @@ -1,28 +1,17 @@ package org.broadinstitute.dsm.model.elastic.migration; -import java.lang.reflect.Field; -import java.util.HashMap; import java.util.List; import java.util.Map; import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; -import org.broadinstitute.dsm.model.NameValue; -import org.broadinstitute.dsm.model.elastic.export.ElasticMappingExportAdapter; -import org.broadinstitute.dsm.model.elastic.export.RequestPayload; -import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; -import org.broadinstitute.dsm.model.elastic.export.generate.MappingGenerator; -import org.broadinstitute.dsm.model.elastic.export.generate.MappingGeneratorFactory; -import org.broadinstitute.dsm.model.elastic.export.parse.BaseParser; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; -import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; import org.broadinstitute.dsm.statics.ESObjectConstants; public class ParticipantDataMigrator extends BaseCollectionMigrator { - public static final String DATA_WITH_ALIAS = "d.data"; private ParticipantDataDao participantDataDao; public ParticipantDataMigrator(String index, String realm) { @@ -36,38 +25,6 @@ protected Map getDataByRealm() { return (Map) participantDataByRealm; } - @Override - public void export() { - exportMapping(); - super.export(); - } - - private void exportMapping() { - Map mapping = new HashMap<>(); - Map dataByRealm = getDataByRealm(); - BaseParser typeParser = new TypeParser(); - try { - Field dataField = ParticipantDataDto.class.getDeclaredField(ParticipantDataDao.DATA); - dataField.setAccessible(true); - for (Map.Entry entry : dataByRealm.entrySet()) { - List participantDatas = (List) entry.getValue(); - for (Object jsonData : participantDatas) { - NameValue nameValue = new NameValue(DATA_WITH_ALIAS, dataField.get(jsonData)); - GeneratorPayload generatorPayload = new GeneratorPayload(nameValue); - MappingGenerator mappingGenerator = (MappingGenerator) new MappingGeneratorFactory().make(new BaseGenerator.PropertyInfo("", true)); - mappingGenerator.setParser(typeParser); - mappingGenerator.setPayload(generatorPayload); - mapping = mappingGenerator.merge(mapping, mappingGenerator.generate()); - } - } - } catch (NoSuchFieldException | IllegalAccessException e) { - throw new RuntimeException(e); - } - ElasticMappingExportAdapter mappingExporter = new ElasticMappingExportAdapter(); - mappingExporter.setRequestPayload(new RequestPayload(index)); - mappingExporter.setSource(mapping); - mappingExporter.export(); - } @Override public void setParser(Parser parser) { diff --git a/src/main/java/org/broadinstitute/dsm/pubsub/DSMtasksSubscription.java b/src/main/java/org/broadinstitute/dsm/pubsub/DSMtasksSubscription.java index e7ed53ffe..bdd571895 100644 --- a/src/main/java/org/broadinstitute/dsm/pubsub/DSMtasksSubscription.java +++ b/src/main/java/org/broadinstitute/dsm/pubsub/DSMtasksSubscription.java @@ -103,7 +103,10 @@ private static void migrateToES(ExportToES.ExportPayload exportPayload) { new DDPInstanceDao().getDDPInstanceByInstanceName(study); maybeDdpInstanceByInstanceName.ifPresent(ddpInstanceDto -> { String index = ddpInstanceDto.getEsParticipantIndex(); - List exportables = Arrays.asList(new MedicalRecordMigrator(index, study), + List exportables = Arrays.asList( + //DynamicFieldsMappingMigrator should be first in the list to make sure that mapping will be exported for first + new DynamicFieldsMappingMigrator(index, study), + new MedicalRecordMigrator(index, study), new OncHistoryDetailsMigrator(index, study), new ParticipantDataMigrator(index, study), new ParticipantMigrator(index, study), diff --git a/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java b/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java index e677cafb5..e4bbf03da 100644 --- a/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java +++ b/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java @@ -6,6 +6,7 @@ public class ESObjectConstants { public static final String PARTICIPANT_DATA_ID = "participantDataId"; + public static final String ADDITIONAL_VALUES_JSON = "additionalValuesJson"; //workflows public static final String ELASTIC_EXPORT_WORKFLOWS = "ELASTIC_EXPORT.workflows"; From 655d9870c504295f690cac03242cf75dad297a9d Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 30 Nov 2021 18:08:52 +0400 Subject: [PATCH 171/405] DDP-7121 | remove unused methods from BaseCollectionMigrator --- .../migration/BaseCollectionMigrator.java | 82 +------------------ .../migration/MedicalRecordMigrator.java | 6 +- 2 files changed, 4 insertions(+), 84 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java index 5c7996718..f0a4925cf 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java @@ -1,32 +1,15 @@ package org.broadinstitute.dsm.model.elastic.migration; -import java.lang.reflect.Field; -import java.util.*; -import java.util.stream.Collectors; - -import org.apache.commons.lang3.StringUtils; -import org.broadinstitute.dsm.db.structure.ColumnName; -import org.broadinstitute.dsm.db.structure.TableName; -import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.elastic.Util; -import org.broadinstitute.dsm.model.elastic.export.ElasticMappingExportAdapter; -import org.broadinstitute.dsm.model.elastic.export.RequestPayload; -import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; -import org.broadinstitute.dsm.model.elastic.export.generate.CollectionMappingGenerator; -import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; -import org.broadinstitute.dsm.model.elastic.export.generate.MappingGenerator; -import org.broadinstitute.dsm.model.elastic.export.generate.Merger; -import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; import org.broadinstitute.dsm.statics.ESObjectConstants; -import static org.broadinstitute.dsm.util.ElasticSearchUtil.PROPERTIES; +import java.util.*; +import java.util.stream.Collectors; public abstract class BaseCollectionMigrator extends BaseMigrator { - private final BaseGenerator collectionMappingGenerator = new CollectionMappingGenerator(); protected List> transformedList; protected Set primaryKeys; - private Map endResult = new HashMap<>(); public BaseCollectionMigrator(String index, String realm, String object) { super(index, realm, object); @@ -40,71 +23,10 @@ public Map generate() { @Override protected void transformObject(Object object) { - List objects = (List) object; - Optional maybeObject = objects.stream().findFirst(); - maybeObject.ifPresent(this::collectPrimaryKeys); - collectionMappingGenerator.setParser(new TypeParser()); - for (Object o: objects) { - Class clazz = o.getClass(); - Field[] declaredFields = clazz.getDeclaredFields(); - TableName table = clazz.getAnnotation(TableName.class); - for (Field field: declaredFields) { - ColumnName columnName = field.getAnnotation(ColumnName.class); - if (columnName == null) continue; - field.setAccessible(true); - String fieldName = field.getName(); - String fieldNameWithAlias = table.alias() + "." + fieldName; - try { - Object fieldValue = field.get(o); - if (fieldValue == null) continue; - NameValue nameValue = new NameValue(fieldNameWithAlias, fieldValue); - GeneratorPayload generatorPayload = new GeneratorPayload(nameValue); - collectionMappingGenerator.setPayload(generatorPayload); - endResult = merge(endResult, collectionMappingGenerator.generate()); - } catch (IllegalAccessException e) { - throw new RuntimeException(e); - } - } - } transformedList = Util.transformObjectCollectionToCollectionMap((List) object); setPrimaryId(); } - private void collectPrimaryKeys(Object obj) { - Class clazz = obj.getClass(); - extractAndCollectPrimaryKey(clazz); - List listFields = getListTypeFields(clazz); - for (Field field : listFields) { - try { - Class parameterizedType = Util.getParameterizedType(field.getGenericType()); - extractAndCollectPrimaryKey(parameterizedType); - } catch (ClassNotFoundException e) { - throw new RuntimeException(e); - } - } - } - - private void extractAndCollectPrimaryKey(Class clazz) { - TableName upperTable = clazz.getAnnotation(TableName.class); - if (hasPrimaryKey(upperTable)) { - this.primaryKeys.add(Util.underscoresToCamelCase(upperTable.primaryKey())); - } - } - - private boolean hasPrimaryKey(TableName table) { - return table != null && StringUtils.isNotBlank(table.primaryKey()); - } - - private List getListTypeFields(Class clazz) { - return Arrays.stream(clazz.getDeclaredFields()) - .filter(this::isFieldListType) - .collect(Collectors.toList()); - } - - public boolean isFieldListType(Field field) { - return List.class.isAssignableFrom(field.getType()); - } - private void setPrimaryId() { for(Map map: transformedList) { List listValueKeys = getListValueKeys(map); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigrator.java index bae55acb9..7e0e56f89 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigrator.java @@ -1,14 +1,12 @@ package org.broadinstitute.dsm.model.elastic.migration; -import java.util.Map; - import org.broadinstitute.dsm.db.MedicalRecord; -import org.broadinstitute.dsm.model.elastic.export.generate.CollectionMappingGenerator; import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; -import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; import org.broadinstitute.dsm.statics.ESObjectConstants; +import java.util.Map; + public class MedicalRecordMigrator extends BaseCollectionMigrator { public MedicalRecordMigrator(String index, String realm) { From 21dd8db4df38fb60fbfe3515ae448447f349f3cf Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 1 Dec 2021 10:37:27 +0400 Subject: [PATCH 172/405] DDP-7121 | change to fetch all field settings --- .../dsm/db/dao/settings/FieldSettingsDao.java | 26 +++---------------- .../DynamicFieldsMappingMigrator.java | 2 +- 2 files changed, 4 insertions(+), 24 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/dao/settings/FieldSettingsDao.java b/src/main/java/org/broadinstitute/dsm/db/dao/settings/FieldSettingsDao.java index 106ef9a00..d7eac3b20 100644 --- a/src/main/java/org/broadinstitute/dsm/db/dao/settings/FieldSettingsDao.java +++ b/src/main/java/org/broadinstitute/dsm/db/dao/settings/FieldSettingsDao.java @@ -12,7 +12,6 @@ import java.util.stream.Collectors; import org.broadinstitute.ddp.db.SimpleResult; -import org.broadinstitute.dsm.db.FieldSettings; import org.broadinstitute.dsm.db.dao.Dao; import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; @@ -36,22 +35,6 @@ public class FieldSettingsDao implements Dao { "changed_by" + " FROM field_settings WHERE ddp_instance_id = ? and (display_type = 'OPTIONS' or display_type = 'RADIO') "; - private static final String SQL_FIELD_SETTINGS_BY_STUDY_NAME = "SELECT " + - "field_settings_id," + - "ddp_instance_id," + - "field_type," + - "column_name," + - "column_display," + - "display_type," + - "possible_values," + - "actions," + - "readonly," + - "order_number," + - "deleted," + - "last_changed," + - "changed_by" + - " FROM field_settings WHERE ddp_instance_id = (SELECT ddp_instance_id FROM ddp_instance WHERE instance_name = ?)"; - private static final String GET_FIELD_SETTINGS = "SELECT " + "field_settings_id," + "ddp_instance_id," + @@ -211,13 +194,11 @@ public List getOptionAndRadioFieldSettingsByInstanceId(int ins return fieldSettingsByOptions; } - public List getFieldSettingsByStudyName(String studyName) { - + public List getAllFieldSettings() { List fieldSettingsByOptions = new ArrayList<>(); SimpleResult results = inTransaction((conn) -> { SimpleResult execResult = new SimpleResult(); - try (PreparedStatement stmt = conn.prepareStatement(SQL_FIELD_SETTINGS_BY_STUDY_NAME)) { - stmt.setString(1, studyName); + try (PreparedStatement stmt = conn.prepareStatement(GET_FIELD_SETTINGS)) { try(ResultSet rs = stmt.executeQuery()) { while (rs.next()) { fieldSettingsByOptions.add( @@ -245,8 +226,7 @@ public List getFieldSettingsByStudyName(String studyName) { return execResult; }); if (results.resultException != null) { - throw new RuntimeException("Error getting fieldSettings for study: " - + studyName, results.resultException); + throw new RuntimeException("Error getting fieldSettings: ", results.resultException); } return fieldSettingsByOptions; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java index 06bf8eda5..493949dc9 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java @@ -40,7 +40,7 @@ public DynamicFieldsMappingMigrator(String index, String study) { @Override public void export() { FieldSettingsDao fieldSettingsDao = FieldSettingsDao.of(); - List fieldSettingsByStudyName = fieldSettingsDao.getFieldSettingsByStudyName(study); + List fieldSettingsByStudyName = fieldSettingsDao.getAllFieldSettings(); for (FieldSettingsDto fieldSettingsDto : fieldSettingsByStudyName) { String fieldType = fieldSettingsDto.getFieldType(); BaseGenerator.PropertyInfo propertyInfo = Util.TABLE_ALIAS_MAPPINGS.get(fieldType); From 42b55dafbb260a72a1bbc917a47d1458fac6404e Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 1 Dec 2021 11:23:33 +0400 Subject: [PATCH 173/405] DDP-7121 | take care of exporting dynamic field data correctly --- .../org/broadinstitute/dsm/model/elastic/Util.java | 6 +++--- .../migration/DynamicFieldsMappingMigrator.java | 11 ++++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 721354bd0..198675bad 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -115,9 +115,9 @@ static Map convertToMap(String fieldName, Object fieldValue) { switch (fieldName) { case "follow_ups": finalResult = new HashMap<>(Map.of(underscoresToCamelCase(fieldName), new Gson().toJson(fieldValue))); -// List> followUps = convertObjectListToMapList(fieldValue); -// finalResult = Map.of(underscoresToCamelCase(fieldName), followUps); break; + case "additional_tissue_value_json": + case "additional_values_json": case "data": Map objectMap = dynamicFieldsSpecialCase(fieldValue); Map transformedMap = new HashMap<>(); @@ -127,7 +127,7 @@ static Map convertToMap(String fieldName, Object fieldValue) { String camelCaseField = underscoresToCamelCase(field); transformedMap.put(camelCaseField, value); } - finalResult = transformedMap; + finalResult = Map.of("dynamicFields", transformedMap); break; default: finalResult = new HashMap<>(Map.of(underscoresToCamelCase(fieldName), fieldValue)); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java index 493949dc9..35bb70802 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java @@ -22,6 +22,7 @@ public class DynamicFieldsMappingMigrator implements Exportable { + public static final String DYNAMIC_FIELDS_WRAPPER_NAME = "dynamicFields"; private final String index; private final String study; public Parser parser; @@ -45,9 +46,9 @@ public void export() { String fieldType = fieldSettingsDto.getFieldType(); BaseGenerator.PropertyInfo propertyInfo = Util.TABLE_ALIAS_MAPPINGS.get(fieldType); if (propertyInfo != null) - buildMapping(fieldSettingsDto, propertyInfo, ESObjectConstants.ADDITIONAL_VALUES_JSON); + buildMapping(fieldSettingsDto, propertyInfo); else - buildMapping(fieldSettingsDto, new BaseGenerator.PropertyInfo(ESObjectConstants.PARTICIPANT_DATA, true), DBConstants.DATA); + buildMapping(fieldSettingsDto, new BaseGenerator.PropertyInfo(ESObjectConstants.PARTICIPANT_DATA, true)); } elasticMappingExportAdapter.setRequestPayload(new RequestPayload(index)); elasticMappingExportAdapter.setSource(buildFinalMapping()); @@ -61,12 +62,12 @@ private Map buildFinalMapping() { return finalMap; } - private void buildMapping(FieldSettingsDto fieldSettingsDto, BaseGenerator.PropertyInfo propertyInfo, String dynamicFieldsWrapperName) { + private void buildMapping(FieldSettingsDto fieldSettingsDto, BaseGenerator.PropertyInfo propertyInfo) { String columnName = Util.underscoresToCamelCase(fieldSettingsDto.getColumnName()); String propertyName = propertyInfo.getPropertyName(); Object typeMap = parser.parse(fieldSettingsDto.getDisplayType()); if (!(propertyMap.containsKey(propertyName))) { - Map additionalValuesJson = new HashMap<>(Map.of(dynamicFieldsWrapperName, new HashMap<>(Map.of(PROPERTIES, new HashMap<>(Map.of(columnName, typeMap)))))); + Map additionalValuesJson = new HashMap<>(Map.of(DYNAMIC_FIELDS_WRAPPER_NAME, new HashMap<>(Map.of(PROPERTIES, new HashMap<>(Map.of(columnName, typeMap)))))); Map wrapperMap = new HashMap<>(); if (propertyInfo.isCollection()) { wrapperMap.put(MappingGenerator.TYPE, MappingGenerator.NESTED); @@ -76,7 +77,7 @@ private void buildMapping(FieldSettingsDto fieldSettingsDto, BaseGenerator.Prope } else { Map outerMap = (Map) propertyMap.get(propertyName); Map outerProperties = (Map) outerMap.get(PROPERTIES); - Map dynamicFieldsJson = (Map) outerProperties.get(dynamicFieldsWrapperName); + Map dynamicFieldsJson = (Map) outerProperties.get(DYNAMIC_FIELDS_WRAPPER_NAME); Map innerProperties = (Map) dynamicFieldsJson.get(PROPERTIES); innerProperties.putIfAbsent(columnName, typeMap); } From 213089fe7888f7a200725abf3b7a8b86f6841b33 Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 1 Dec 2021 16:18:44 +0400 Subject: [PATCH 174/405] DDP-7121 | add json ignored property to ESAddress, convert CHECKBOX type to boolean type --- .../dsm/model/elastic/ESAddress.java | 2 ++ .../migration/DynamicFieldsMappingMigrator.java | 2 ++ .../DynamicFieldsMappingMigratorTest.java | 16 ++++++++++++++++ 3 files changed, 20 insertions(+) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/ESAddress.java b/src/main/java/org/broadinstitute/dsm/model/elastic/ESAddress.java index 687f90b11..8d19b1c2c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/ESAddress.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/ESAddress.java @@ -1,9 +1,11 @@ package org.broadinstitute.dsm.model.elastic; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.google.gson.annotations.SerializedName; import lombok.Setter; @Setter +@JsonIgnoreProperties(ignoreUnknown = true) public class ESAddress { @SerializedName("street1") diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java index 35bb70802..a16de331c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java @@ -102,6 +102,8 @@ public Object parse(String value) { Object parsedValue; if ("DATE".equals(value)) { parsedValue = forDate(value); + } else if ("CHECKBOX".equals(value)) { + parsedValue = forBoolean(value); } else { parsedValue = forString(value); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java index 81bf5136e..b0b371db0 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java @@ -6,6 +6,9 @@ import org.junit.BeforeClass; import org.junit.Test; +import java.util.Arrays; +import java.util.List; + public class DynamicFieldsMappingMigratorTest { @BeforeClass @@ -15,6 +18,19 @@ public static void setUp() { @Test public void testExport() { +// final String index = "participants_structured.cmi.angio"; +// final String study = "angio"; +// List exportables = Arrays.asList( +// //DynamicFieldsMappingMigrator should be first in the list to make sure that mapping will be exported for first +// new DynamicFieldsMappingMigrator(index, study), +// new MedicalRecordMigrator(index, study), +// new OncHistoryDetailsMigrator(index, study), +// new ParticipantDataMigrator(index, study), +// new ParticipantMigrator(index, study), +// new KitRequestShippingMigrator(index, study), +// new TissueMigrator(index, study)); +// exportables.forEach(Exportable::export); + Exportable exportable = new DynamicFieldsMappingMigrator("participants_structured.cmi.angio", "angio"); exportable.export(); } From c78d62e7b8be55ce327dc8365d1d29a99affc433 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 1 Dec 2021 16:51:51 +0400 Subject: [PATCH 175/405] DDP-7121 | minor refactorings --- .../model/elastic/migration/DynamicFieldsMappingMigrator.java | 4 ++-- .../java/org/broadinstitute/dsm/util/ElasticSearchUtil.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java index a16de331c..683368eca 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java @@ -67,12 +67,12 @@ private void buildMapping(FieldSettingsDto fieldSettingsDto, BaseGenerator.Prope String propertyName = propertyInfo.getPropertyName(); Object typeMap = parser.parse(fieldSettingsDto.getDisplayType()); if (!(propertyMap.containsKey(propertyName))) { - Map additionalValuesJson = new HashMap<>(Map.of(DYNAMIC_FIELDS_WRAPPER_NAME, new HashMap<>(Map.of(PROPERTIES, new HashMap<>(Map.of(columnName, typeMap)))))); + Map dynamicFields = new HashMap<>(Map.of(DYNAMIC_FIELDS_WRAPPER_NAME, new HashMap<>(Map.of(PROPERTIES, new HashMap<>(Map.of(columnName, typeMap)))))); Map wrapperMap = new HashMap<>(); if (propertyInfo.isCollection()) { wrapperMap.put(MappingGenerator.TYPE, MappingGenerator.NESTED); } - wrapperMap.put(PROPERTIES, additionalValuesJson); + wrapperMap.put(PROPERTIES, dynamicFields); propertyMap.put(propertyName, wrapperMap); } else { Map outerMap = (Map) propertyMap.get(propertyName); diff --git a/src/main/java/org/broadinstitute/dsm/util/ElasticSearchUtil.java b/src/main/java/org/broadinstitute/dsm/util/ElasticSearchUtil.java index 58fd44aec..16afa1597 100644 --- a/src/main/java/org/broadinstitute/dsm/util/ElasticSearchUtil.java +++ b/src/main/java/org/broadinstitute/dsm/util/ElasticSearchUtil.java @@ -133,7 +133,7 @@ public static synchronized void initClient() { } } - private static void fetchFieldMappings() { + private static synchronized void fetchFieldMappings() { GetMappingsRequest request = new GetMappingsRequest(); request.indices(PARTICIPANTS_STRUCTURED_ANY); try { From ed69475d58f1e51f90562cec80d4a9cd170b2557 Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 1 Dec 2021 17:16:44 +0400 Subject: [PATCH 176/405] DDP-7121 | add DynamicFieldsMappingGenerator for generating mapping for patch --- .../export/generate/MappingGenerator.java | 5 ++++ .../generate/MappingGeneratorFactory.java | 13 ++++++++-- .../generate/SingleMappingGenerator.java | 8 +++++++ .../DynamicFieldsMappingGenerator.java | 24 +++++++++++++++++++ 4 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/DynamicFieldsMappingGenerator.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java index b5c6b3c70..b4b8936ba 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java @@ -15,6 +15,7 @@ abstract public class MappingGenerator extends BaseGenerator { public static final String TYPE = "type"; public static final String NESTED = "nested"; public static final String TYPE_KEYWORD = "keyword"; + protected Generator dynamicFieldsMappingGenerator; public MappingGenerator(Parser parser, GeneratorPayload generatorPayload) { super(parser, generatorPayload); @@ -22,6 +23,10 @@ public MappingGenerator(Parser parser, GeneratorPayload generatorPayload) { public MappingGenerator() {} + public MappingGenerator(Generator dynamicFieldsMappingGenerator) { + this.dynamicFieldsMappingGenerator = dynamicFieldsMappingGenerator; + } + @Override public Map generate() { logger.info("preparing mapping to upsert"); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorFactory.java index ede16cfc6..dde3df255 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorFactory.java @@ -4,10 +4,19 @@ public class MappingGeneratorFactory implements GeneratorFactory { @Override public BaseGenerator make(BaseGenerator.PropertyInfo propertyInfo) { BaseGenerator generator; + if (propertyInfo.isCollection()) { - generator = new CollectionMappingGenerator(); + if ("additionalValuesJson".equals(propertyInfo.getPropertyName()) || "data".equals(propertyInfo.getPropertyName())) { + generator = new CollectionMappingGenerator(new DynamicFieldsMappingGenerator()); + } else { + generator = new CollectionMappingGenerator(); + } } else { - generator = new SingleMappingGenerator(); + if ("additionalValuesJson".equals(propertyInfo.getPropertyName()) || "data".equals(propertyInfo.getPropertyName())) { + generator = new SingleMappingGenerator(new DynamicFieldsMappingGenerator()); + } else { + generator = new SingleMappingGenerator(); + } } return generator; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java index 030552678..42a1cb0ee 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java @@ -1,10 +1,13 @@ package org.broadinstitute.dsm.model.elastic.export.generate; +import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; +import org.jruby.ir.operands.Hash; import java.util.HashMap; import java.util.Map; +import java.util.Objects; public class SingleMappingGenerator extends MappingGenerator { @@ -16,6 +19,11 @@ public SingleMappingGenerator() {} @Override protected Map getElement(Object type) { + if (!Objects.nonNull(dynamicFieldsMappingGenerator)) { + dynamicFieldsMappingGenerator.setPayload(generatorPayload); + dynamicFieldsMappingGenerator.setParser(parser); + return dynamicFieldsMappingGenerator.generate(); + } return new HashMap<>(Map.of(Util.underscoresToCamelCase(getDBElement().getColumnName()), type)); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/DynamicFieldsMappingGenerator.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/DynamicFieldsMappingGenerator.java new file mode 100644 index 000000000..a1949a29a --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/DynamicFieldsMappingGenerator.java @@ -0,0 +1,24 @@ +package org.broadinstitute.dsm.model.elastic.export.generate; + +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; + +import java.util.Map; + +public class DynamicFieldsMappingGenerator implements Generator { + + + @Override + public Map generate() { + return null; + } + + @Override + public void setParser(Parser parser) { + + } + + @Override + public void setPayload(GeneratorPayload generatorPayload) { + + } +} From 05cd55f9011cd0c4c4d342c9534ae6135d729d0e Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 1 Dec 2021 17:52:00 +0400 Subject: [PATCH 177/405] DDP-7121 | added code for dynamicfieldmappinggenerator --- .../model/elastic/export/ExportFacade.java | 1 + .../elastic/export/ExportFacadePayload.java | 4 +++ .../export/generate/BaseGenerator.java | 2 +- .../generate/CollectionMappingGenerator.java | 12 +++++++ .../DynamicFieldsMappingGenerator.java | 31 +++++++++++++++++++ .../export/generate/GeneratorPayload.java | 5 +++ .../generate/MappingGeneratorFactory.java | 4 +-- .../generate/SingleMappingGenerator.java | 6 +++- .../DynamicFieldsMappingGenerator.java | 24 -------------- 9 files changed, 61 insertions(+), 28 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/DynamicFieldsMappingGenerator.java delete mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/DynamicFieldsMappingGenerator.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 7a2451f38..4d46922e1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -42,6 +42,7 @@ public void export() { private void upsertMapping() { BaseGenerator.PropertyInfo propertyInfo = getPropertyInfo(); GeneratorFactory generatorFactory = new MappingGeneratorFactory(); + propertyInfo.setFieldName(exportFacadePayload.getFieldName()); generator = generatorFactory.make(propertyInfo); generator.setParser(new TypeParser()); generator.setPayload(exportFacadePayload.getGeneratorPayload()); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java index 16a5bf180..58f436644 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java @@ -47,4 +47,8 @@ public GeneratorPayload getGeneratorPayload() { public String getFieldNameWithAlias() { return generatorPayload.getName(); } + + public String getFieldName() { + return generatorPayload.getFieldName(); + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java index b2a75063a..1313b5312 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java @@ -71,7 +71,7 @@ public String getPropertyName() { } public String getFieldName() { - return Util.underscoresToCamelCase(getDBElement().getColumnName()); + return generatorPayload.getFieldName(); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java index 0fca6b498..46230b764 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java @@ -5,6 +5,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.Objects; public class CollectionMappingGenerator extends MappingGenerator { @@ -14,8 +15,19 @@ public CollectionMappingGenerator(Parser parser, GeneratorPayload generatorPaylo public CollectionMappingGenerator() {} + public CollectionMappingGenerator(Generator dynamicFieldsMappingGenerator) { + super(dynamicFieldsMappingGenerator); + } + @Override protected Map getElement(Object type) { + if (!Objects.nonNull(dynamicFieldsMappingGenerator)) { + dynamicFieldsMappingGenerator.setPayload(generatorPayload); + dynamicFieldsMappingGenerator.setParser(parser); + HashMap elementWithIdAndType = new HashMap<>(Map.of(Util.ID, new HashMap<>(Map.of(TYPE, TYPE_KEYWORD)))); + elementWithIdAndType.putAll(dynamicFieldsMappingGenerator.generate()); + return elementWithIdAndType; + } return new HashMap<>(Map.of( Util.ID, new HashMap<>(Map.of(TYPE, TYPE_KEYWORD)), Util.underscoresToCamelCase(getDBElement().getColumnName()), type diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/DynamicFieldsMappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/DynamicFieldsMappingGenerator.java new file mode 100644 index 000000000..25ec83b05 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/DynamicFieldsMappingGenerator.java @@ -0,0 +1,31 @@ +package org.broadinstitute.dsm.model.elastic.export.generate; + +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; + +import java.util.HashMap; +import java.util.Map; + +public class DynamicFieldsMappingGenerator implements Generator { + + + private Parser parser; + private GeneratorPayload generatorPayload; + + + @Override + public Map generate() { + Object type = parser.parse(String.valueOf(generatorPayload.getValue())); + Map fieldWithType = new HashMap<>(Map.of(generatorPayload.getFieldName(), type)); + return new HashMap<>(Map.of("dynamicFields", Map.of("properties", fieldWithType))); + } + + @Override + public void setParser(Parser parser) { + this.parser = parser; + } + + @Override + public void setPayload(GeneratorPayload generatorPayload) { + this.generatorPayload = generatorPayload; + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorPayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorPayload.java index 9023e3809..1dbd6fdc9 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorPayload.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorPayload.java @@ -1,6 +1,7 @@ package org.broadinstitute.dsm.model.elastic.export.generate; import org.broadinstitute.dsm.model.NameValue; +import org.broadinstitute.dsm.model.elastic.Util; public class GeneratorPayload { @@ -27,4 +28,8 @@ public NameValue getNameValue() { public int getRecordId() { return recordId; } + + public String getFieldName() { + return Util.underscoresToCamelCase(Util.getDBElement(getName()).getColumnName()); + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorFactory.java index dde3df255..549687816 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorFactory.java @@ -6,13 +6,13 @@ public BaseGenerator make(BaseGenerator.PropertyInfo propertyInfo) { BaseGenerator generator; if (propertyInfo.isCollection()) { - if ("additionalValuesJson".equals(propertyInfo.getPropertyName()) || "data".equals(propertyInfo.getPropertyName())) { + if ("additionalValuesJson".equals(propertyInfo.getFieldName()) || "data".equals(propertyInfo.getFieldName())) { generator = new CollectionMappingGenerator(new DynamicFieldsMappingGenerator()); } else { generator = new CollectionMappingGenerator(); } } else { - if ("additionalValuesJson".equals(propertyInfo.getPropertyName()) || "data".equals(propertyInfo.getPropertyName())) { + if ("additionalValuesJson".equals(propertyInfo.getFieldName()) || "data".equals(propertyInfo.getFieldName())) { generator = new SingleMappingGenerator(new DynamicFieldsMappingGenerator()); } else { generator = new SingleMappingGenerator(); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java index 42a1cb0ee..7c931dea6 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java @@ -17,6 +17,10 @@ public SingleMappingGenerator(Parser parser, GeneratorPayload generatorPayload) public SingleMappingGenerator() {} + public SingleMappingGenerator(Generator dynamicFieldsMappingGenerator) { + super(dynamicFieldsMappingGenerator); + } + @Override protected Map getElement(Object type) { if (!Objects.nonNull(dynamicFieldsMappingGenerator)) { @@ -24,7 +28,7 @@ protected Map getElement(Object type) { dynamicFieldsMappingGenerator.setParser(parser); return dynamicFieldsMappingGenerator.generate(); } - return new HashMap<>(Map.of(Util.underscoresToCamelCase(getDBElement().getColumnName()), type)); + return new HashMap<>(Map.of(getFieldName(), type)); } @Override diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/DynamicFieldsMappingGenerator.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/DynamicFieldsMappingGenerator.java deleted file mode 100644 index a1949a29a..000000000 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/DynamicFieldsMappingGenerator.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.broadinstitute.dsm.model.elastic.export.generate; - -import org.broadinstitute.dsm.model.elastic.export.parse.Parser; - -import java.util.Map; - -public class DynamicFieldsMappingGenerator implements Generator { - - - @Override - public Map generate() { - return null; - } - - @Override - public void setParser(Parser parser) { - - } - - @Override - public void setPayload(GeneratorPayload generatorPayload) { - - } -} From 460d6dc23edf2a66a816161dbe5137408fdf7969 Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 2 Dec 2021 10:30:58 +0400 Subject: [PATCH 178/405] DDP-7121 | add "properties" to dynamicFields" --- .../generate/CollectionSourceGenerator.java | 2 +- .../export/generate/MappingGenerator.java | 6 ++-- .../DynamicFieldsMappingMigratorTest.java | 28 +++++++++---------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java index ebe1efc81..5ca19bcb4 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java @@ -27,7 +27,7 @@ public Object construct() { logger.info("Constructing nested data"); Map collectionMap = new HashMap<>(); collectionMap.put(Util.ID, generatorPayload.getRecordId()); - Map mapWithParsedObjects = parseJsonValuesToObject(); + Map mapWithParsedObjects = new HashMap<>(Map.of("dynamicFields", parseJsonValuesToObject())); collectionMap.putAll(mapWithParsedObjects); return List.of(collectionMap); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java index b4b8936ba..30075f126 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java @@ -7,6 +7,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.Objects; abstract public class MappingGenerator extends BaseGenerator { @@ -50,9 +51,8 @@ protected Map parseJson() { Object eachType = parser.parse(String.valueOf(entry.getValue())); resultMap.put(Util.underscoresToCamelCase(entry.getKey()), eachType); } - return new HashMap<>(Map.of(Util.underscoresToCamelCase(getDBElement().getColumnName()), - new HashMap<>(Map.of(PROPERTIES, - resultMap)))); + Map returnMap = new HashMap<>(Map.of("dynamicFields", new HashMap<>(Map.of(PROPERTIES, resultMap)))); + return returnMap; } @Override diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java index b0b371db0..19eaa1cac 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java @@ -18,20 +18,20 @@ public static void setUp() { @Test public void testExport() { -// final String index = "participants_structured.cmi.angio"; -// final String study = "angio"; -// List exportables = Arrays.asList( -// //DynamicFieldsMappingMigrator should be first in the list to make sure that mapping will be exported for first -// new DynamicFieldsMappingMigrator(index, study), -// new MedicalRecordMigrator(index, study), -// new OncHistoryDetailsMigrator(index, study), -// new ParticipantDataMigrator(index, study), -// new ParticipantMigrator(index, study), -// new KitRequestShippingMigrator(index, study), -// new TissueMigrator(index, study)); -// exportables.forEach(Exportable::export); + final String index = "participants_structured.cmi.angio"; + final String study = "angio"; + List exportables = Arrays.asList( + //DynamicFieldsMappingMigrator should be first in the list to make sure that mapping will be exported for first + new DynamicFieldsMappingMigrator(index, study), + new MedicalRecordMigrator(index, study), + new OncHistoryDetailsMigrator(index, study), + new ParticipantDataMigrator(index, study), + new ParticipantMigrator(index, study), + new KitRequestShippingMigrator(index, study), + new TissueMigrator(index, study)); + exportables.forEach(Exportable::export); - Exportable exportable = new DynamicFieldsMappingMigrator("participants_structured.cmi.angio", "angio"); - exportable.export(); +// Exportable exportable = new DynamicFieldsMappingMigrator("participants_structured.cmi.angio", "angio"); +// exportable.export(); } } \ No newline at end of file From a537e6131b8570aa35d30acbf3e4f4283b206319 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 2 Dec 2021 11:06:44 +0400 Subject: [PATCH 179/405] DPP-7121 | remove unnecessary sets of ID --- .../dsm/model/elastic/Util.java | 1 - .../generate/CollectionMappingGenerator.java | 22 ++------- .../generate/CollectionSourceGenerator.java | 8 +--- .../generate/MappingGeneratorFactory.java | 19 ++------ .../export/process/CollectionProcessor.java | 6 +-- .../migration/BaseCollectionMigrator.java | 47 ------------------- .../export/generate/MappingGeneratorTest.java | 4 +- 7 files changed, 11 insertions(+), 96 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 198675bad..49a24d35e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -33,7 +33,6 @@ public class Util { public static final int FIRST_ELEMENT_INDEX = 0; public static final String UNDERSCORE_SEPARATOR = "_"; public static final String DOC = "_doc"; - public static final String ID = "id"; private static final Pattern CAMEL_CASE_REGEX = Pattern.compile("(([a-z])+([A-z]))*"); public static final Gson GSON = new Gson(); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java index 46230b764..8395aff0b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java @@ -1,11 +1,9 @@ package org.broadinstitute.dsm.model.elastic.export.generate; -import org.broadinstitute.dsm.model.elastic.Util; -import org.broadinstitute.dsm.model.elastic.export.parse.Parser; - import java.util.HashMap; import java.util.Map; -import java.util.Objects; + +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; public class CollectionMappingGenerator extends MappingGenerator { @@ -15,23 +13,9 @@ public CollectionMappingGenerator(Parser parser, GeneratorPayload generatorPaylo public CollectionMappingGenerator() {} - public CollectionMappingGenerator(Generator dynamicFieldsMappingGenerator) { - super(dynamicFieldsMappingGenerator); - } - @Override protected Map getElement(Object type) { - if (!Objects.nonNull(dynamicFieldsMappingGenerator)) { - dynamicFieldsMappingGenerator.setPayload(generatorPayload); - dynamicFieldsMappingGenerator.setParser(parser); - HashMap elementWithIdAndType = new HashMap<>(Map.of(Util.ID, new HashMap<>(Map.of(TYPE, TYPE_KEYWORD)))); - elementWithIdAndType.putAll(dynamicFieldsMappingGenerator.generate()); - return elementWithIdAndType; - } - return new HashMap<>(Map.of( - Util.ID, new HashMap<>(Map.of(TYPE, TYPE_KEYWORD)), - Util.underscoresToCamelCase(getDBElement().getColumnName()), type - )); + return new HashMap<>(Map.of(getFieldName(), type)); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java index 5ca19bcb4..072a67779 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java @@ -25,16 +25,12 @@ public CollectionSourceGenerator() { @Override public Object construct() { logger.info("Constructing nested data"); - Map collectionMap = new HashMap<>(); - collectionMap.put(Util.ID, generatorPayload.getRecordId()); Map mapWithParsedObjects = new HashMap<>(Map.of("dynamicFields", parseJsonValuesToObject())); - collectionMap.putAll(mapWithParsedObjects); - return List.of(collectionMap); + return List.of(mapWithParsedObjects); } @Override protected Object getElement(Object element) { - return List.of(new HashMap<>(Map.of(Util.underscoresToCamelCase(getDBElement().getColumnName()), element, - Util.ID, generatorPayload.getRecordId()))); + return List.of(new HashMap<>(Map.of(getFieldName(), element))); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorFactory.java index 549687816..22fd150c5 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorFactory.java @@ -3,21 +3,8 @@ public class MappingGeneratorFactory implements GeneratorFactory { @Override public BaseGenerator make(BaseGenerator.PropertyInfo propertyInfo) { - BaseGenerator generator; - - if (propertyInfo.isCollection()) { - if ("additionalValuesJson".equals(propertyInfo.getFieldName()) || "data".equals(propertyInfo.getFieldName())) { - generator = new CollectionMappingGenerator(new DynamicFieldsMappingGenerator()); - } else { - generator = new CollectionMappingGenerator(); - } - } else { - if ("additionalValuesJson".equals(propertyInfo.getFieldName()) || "data".equals(propertyInfo.getFieldName())) { - generator = new SingleMappingGenerator(new DynamicFieldsMappingGenerator()); - } else { - generator = new SingleMappingGenerator(); - } - } - return generator; + return propertyInfo.isCollection() + ? new CollectionMappingGenerator() + : new SingleMappingGenerator(); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java index 6fb983155..fa8bfe03f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java @@ -82,10 +82,8 @@ protected List> updateIfExistsOrPut(List } private boolean isExistingRecord(Map eachRecord) { - if (!eachRecord.containsKey(Util.underscoresToCamelCase(primaryKey)) && !eachRecord.containsKey(Util.ID)) return false; - double id = eachRecord.containsKey(Util.ID) - ? Double.parseDouble(String.valueOf(eachRecord.get(Util.ID))) - : Double.parseDouble(String.valueOf(eachRecord.get(Util.underscoresToCamelCase(primaryKey)))); + if (!eachRecord.containsKey(Util.underscoresToCamelCase(primaryKey))) return false; + double id = Double.parseDouble(String.valueOf(eachRecord.get(Util.underscoresToCamelCase(primaryKey)))); return id == (double) recordId; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java index f0a4925cf..6af5e7184 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java @@ -24,52 +24,5 @@ public Map generate() { @Override protected void transformObject(Object object) { transformedList = Util.transformObjectCollectionToCollectionMap((List) object); - setPrimaryId(); - } - - private void setPrimaryId() { - for(Map map: transformedList) { - List listValueKeys = getListValueKeys(map); - for (String key : listValueKeys) { - List> listValue = (List>) map.get(key); - Optional maybePrimary = getPrimaryKeyFromList(listValue); - maybePrimary.ifPresent(prKey -> { - for (Map stringObjectMap : listValue) { - putPrimaryId(stringObjectMap, prKey); - } - }); - } - getPrimaryKey(map) - .ifPresent(outerKey -> putPrimaryId(map, outerKey)); - } - } - - private List getListValueKeys(Map map) { - return map.entrySet().stream() - .filter(this::isMapValueListType) - .map(Map.Entry::getKey) - .collect(Collectors.toList()); - } - - private boolean isMapValueListType(Map.Entry entry) { - return entry.getValue() instanceof List; - } - - private Optional getPrimaryKeyFromList(List> listValue) { - for(Map eachValue: listValue) { - Optional maybePrimaryKey = getPrimaryKey(eachValue); - if (maybePrimaryKey.isPresent()) return maybePrimaryKey; - } - return Optional.empty(); - } - - private Optional getPrimaryKey(Map map) { - return map.keySet().stream() - .filter(outerKey -> primaryKeys.contains(outerKey)) - .findFirst(); - } - - private void putPrimaryId(Map map, String outerKey) { - map.put(Util.ID, map.get(outerKey)); } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java index 44ac28610..e309602da 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java @@ -87,9 +87,7 @@ public void generateMapping() { Map resultMap = generator.generate(); Map dsmLevelProperty = Map.of(generator.getPropertyName(), Map.of( MappingGenerator.TYPE, MappingGenerator.NESTED, - PROPERTIES, Map.of(Util.underscoresToCamelCase(TestPatchUtil.MEDICAL_RECORD_COLUMN), Map.of(MappingGenerator.TYPE, "date"), - Util.ID, Map.of(MappingGenerator.TYPE, MappingGenerator.TYPE_KEYWORD) - ))); + PROPERTIES, Map.of(Util.underscoresToCamelCase(TestPatchUtil.MEDICAL_RECORD_COLUMN), Map.of(MappingGenerator.TYPE, "date")))); Map dsmLevelProperties = Map.of(PROPERTIES, dsmLevelProperty); Map dsmLevel = Map.of(MappingGenerator.DSM_OBJECT, dsmLevelProperties); Map topLevel = Map.of(PROPERTIES, dsmLevel); From ace1db70a187f2a0c000d6be59a30a8a825916dc Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 2 Dec 2021 11:31:48 +0400 Subject: [PATCH 180/405] DDP-7121 | start implementing the conversion from string to map for dynamic fields while fetching data from ES --- .../broadinstitute/dsm/db/MedicalRecord.java | 5 ++++ .../org/broadinstitute/dsm/db/Tissue.java | 6 ++-- .../dsm/model/elastic/ESDsm.java | 2 +- .../model/elastic/search/ElasticSearch.java | 30 +++++++++++++++++++ 4 files changed, 39 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java index 8bb316e92..c8ca2b7b6 100644 --- a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java +++ b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java @@ -334,6 +334,11 @@ public boolean isFollowUpRequired() { @ColumnName (DBConstants.ADDITIONAL_VALUES) private String additionalValuesJson; + @JsonProperty("dynamicFields") + public String getAdditionalValuesJson() { + return additionalValuesJson; + } + private boolean reviewMedicalRecord; @TableName ( diff --git a/src/main/java/org/broadinstitute/dsm/db/Tissue.java b/src/main/java/org/broadinstitute/dsm/db/Tissue.java index f403267d5..bf156dc1f 100644 --- a/src/main/java/org/broadinstitute/dsm/db/Tissue.java +++ b/src/main/java/org/broadinstitute/dsm/db/Tissue.java @@ -102,7 +102,7 @@ public class Tissue { private String firstSmId; @ColumnName (DBConstants.ADDITIONAL_TISSUE_VALUES) - private String additionalTissueValueJson; + private String additionalValuesJson; @ColumnName (DBConstants.TISSUE_RETURN_DATE) @DbDateConversion(SqlDateConverter.STRING_DAY) @@ -137,7 +137,7 @@ public class Tissue { public Tissue(String tissueId, String oncHistoryDetailId, String notes, Integer countReceived, String tissueType, String tissueSite, String tumorType, String hE, String pathologyReport, String collaboratorSampleId, String blockSent, String scrollsReceived, String skId, String smId, String sentGp, String firstSmId, - String additionalTissueValueJson, String expectedReturn, String returnDate, + String additionalValuesJson, String expectedReturn, String returnDate, String returnFedexId, String shlWorkNumber, String tumorPercentage, String tissueSequence, Integer scrollsCount, Integer ussCount, Integer blocksCount, Integer hECount) { this.tissueId = tissueId; @@ -156,7 +156,7 @@ public Tissue(String tissueId, String oncHistoryDetailId, String notes, Integer this.smId = smId; this.sentGp = sentGp; this.firstSmId = firstSmId; - this.additionalTissueValueJson = additionalTissueValueJson; + this.additionalValuesJson = additionalValuesJson; this.expectedReturn = expectedReturn; this.returnDate = returnDate; this.returnFedexId = returnFedexId; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java index 5b261a8e2..50c0ad00d 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java @@ -41,7 +41,7 @@ public class ESDsm { Object diagnosisYear; @SerializedName(ESObjectConstants.TISSUE_RECORDS) - List> tissueRecords; + List> tissueRecords; // todo: change to Tissue @SerializedName(ESObjectConstants.MEDICAL_RECORDS) List medicalRecords; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java index b6852895a..c4fb0545d 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java @@ -2,6 +2,7 @@ import java.io.IOException; import java.io.StringReader; +import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -21,6 +22,7 @@ import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.MedicalRecord; import org.broadinstitute.dsm.model.FollowUp; +import org.broadinstitute.dsm.model.elastic.ESDsm; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.broadinstitute.dsm.util.ParticipantUtil; @@ -78,6 +80,34 @@ private static ElasticSearchParticipantDto serialize(Map sourceM ElasticSearchParticipantDto elasticSearchParticipantDto = null; Object dsm = sourceMap.get(DSM); if (!Objects.isNull(dsm)) { + + Map dsmLevel = (Map) sourceMap.get(DSM); + for (Map.Entry entry: dsmLevel.entrySet()) { + String key = entry.getKey(); + Object value = entry.getValue(); + if (value instanceof List) { + List> objects = (List>) value; + try { + Field property = ESDsm.class.getDeclaredField(key); + Class propertyType = Util.getParameterizedType(property.getGenericType()); + Field additionalValuesJson = propertyType.getDeclaredField("additionalValuesJson"); + for (Map object : objects) { + Object dynamicFields = object.get("dynamicFields"); + + + } + + } catch (NoSuchFieldException | ClassNotFoundException e) { + e.printStackTrace(); + } + + + } else { + + } + + } + List> medicalRecords = (List>)((Map) sourceMap.get(DSM)).get("medicalRecords"); if (!Objects.isNull(medicalRecords)) { for (Map medicalRecord : medicalRecords) { From c93d518f86360cf999bbef09d74b5ad9c1b145c5 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 2 Dec 2021 12:18:12 +0400 Subject: [PATCH 181/405] DDP-7121 | adding script for remapping dynamicFields --- .../model/elastic/search/ElasticSearch.java | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java index c4fb0545d..2adb0f586 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java @@ -11,6 +11,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import com.fasterxml.jackson.core.JsonProcessingException; @@ -79,25 +80,32 @@ public static Optional parseSourceMap(Map sourceMap) { ElasticSearchParticipantDto elasticSearchParticipantDto = null; Object dsm = sourceMap.get(DSM); + ObjectMapper objectMapper = new ObjectMapper(); if (!Objects.isNull(dsm)) { Map dsmLevel = (Map) sourceMap.get(DSM); for (Map.Entry entry: dsmLevel.entrySet()) { - String key = entry.getKey(); - Object value = entry.getValue(); + Map clonedProperty = new HashMap<>(); + String key = entry.getKey(); //medicalRecords + Object value = entry.getValue(); //medicalRecords: [] if (value instanceof List) { List> objects = (List>) value; try { Field property = ESDsm.class.getDeclaredField(key); Class propertyType = Util.getParameterizedType(property.getGenericType()); - Field additionalValuesJson = propertyType.getDeclaredField("additionalValuesJson"); + propertyType.getDeclaredField("additionalValuesJson"); + List> newObjects = new ArrayList<>(); for (Map object : objects) { - Object dynamicFields = object.get("dynamicFields"); - - + Map clonedMap = new HashMap<>(object); + Object dynamicFields = clonedMap.get("dynamicFields"); + String dynamicFieldsValueAsJson = Objects.isNull(dynamicFields) + ? "" + : objectMapper.writeValueAsString(dynamicFields); + clonedMap.put("dynamicFields", dynamicFieldsValueAsJson); + newObjects.add(clonedMap); } - - } catch (NoSuchFieldException | ClassNotFoundException e) { + clonedProperty.put(key, newObjects); + } catch (NoSuchFieldException | ClassNotFoundException | JsonProcessingException e) { e.printStackTrace(); } @@ -105,7 +113,7 @@ private static ElasticSearchParticipantDto serialize(Map sourceM } else { } - + dsmLevel.put(key, clonedProperty); } List> medicalRecords = (List>)((Map) sourceMap.get(DSM)).get("medicalRecords"); @@ -114,7 +122,7 @@ private static ElasticSearchParticipantDto serialize(Map sourceM if (StringUtils.isNotBlank((String) medicalRecord.get("followUps"))) { String followUps = (String) medicalRecord.get("followUps"); try { - medicalRecord.put("followUps", new ObjectMapper().readValue(followUps, new TypeReference>>(){})); + medicalRecord.put("followUps", objectMapper.readValue(followUps, new TypeReference>>(){})); } catch (IOException e) { throw new RuntimeException(e); } @@ -136,13 +144,13 @@ private static ElasticSearchParticipantDto serialize(Map sourceM // } else { // elasticSearchParticipantDto = GSON.fromJson(GSON.toJson(sourceMap), ElasticSearchParticipantDto.class); // } - elasticSearchParticipantDto = new ObjectMapper().convertValue(sourceMap, ElasticSearchParticipantDto.class); + elasticSearchParticipantDto = objectMapper.convertValue(sourceMap, ElasticSearchParticipantDto.class); } else { - elasticSearchParticipantDto = new ObjectMapper().convertValue(sourceMap, ElasticSearchParticipantDto.class); + elasticSearchParticipantDto = objectMapper.convertValue(sourceMap, ElasticSearchParticipantDto.class); } } else { - elasticSearchParticipantDto = new ObjectMapper().convertValue(sourceMap, ElasticSearchParticipantDto.class); + elasticSearchParticipantDto = objectMapper.convertValue(sourceMap, ElasticSearchParticipantDto.class); } return elasticSearchParticipantDto; // boolean hasMedicalRecord = false; From a09e049177eb39f09dd1889c9adc8a0797c7ba3b Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 2 Dec 2021 16:59:55 +0400 Subject: [PATCH 182/405] DDP-7121 | continue refactoring serialize() method in ElasticSearch.java --- .../dsm/model/elastic/Util.java | 5 +- .../model/elastic/search/ElasticSearch.java | 126 +++++++++--------- .../dsm/model/elastic/UtilTest.java | 5 + 3 files changed, 68 insertions(+), 68 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 49a24d35e..ea41efc66 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -179,8 +179,9 @@ public static Class getParameterizedType(Type genericType) throws ClassNotFou String typeAsString = genericType.toString(); String[] types = typeAsString.contains("<") ? typeAsString.split("<") : typeAsString.split("\\[L"); if (types.length < 2) { - class NullClass {} - return NullClass.class; +// class NullClass {} +// return NullClass.class; + return (Class) genericType; } String parameterizedType = types[1]; parameterizedType = parameterizedType.replace(">", ""); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java index 2adb0f586..84c0f49aa 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java @@ -14,6 +14,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; @@ -77,23 +78,34 @@ public static Optional parseSourceMap(Map sourceMap) { + private static ElasticSearchParticipantDto serialize(Map sourceMap) throws NoSuchFieldException, ClassNotFoundException { ElasticSearchParticipantDto elasticSearchParticipantDto = null; Object dsm = sourceMap.get(DSM); ObjectMapper objectMapper = new ObjectMapper(); if (!Objects.isNull(dsm)) { - Map dsmLevel = (Map) sourceMap.get(DSM); + Map clonedProperty = new HashMap<>(); for (Map.Entry entry: dsmLevel.entrySet()) { - Map clonedProperty = new HashMap<>(); String key = entry.getKey(); //medicalRecords Object value = entry.getValue(); //medicalRecords: [] + Field property = ESDsm.class.getDeclaredField(key); + Class propertyType = Util.getParameterizedType(property.getGenericType()); + Field[] declaredFields = propertyType.getDeclaredFields(); + Optional maybeDynamicFields = Arrays.stream(declaredFields) + .filter(field -> field.getAnnotation(JsonProperty.class).value().equals("dynamicFields")) + .findFirst(); + if (hasDynamicFields) { + if (value instanceof List) { + + } else { + + } + } if (value instanceof List) { List> objects = (List>) value; try { - Field property = ESDsm.class.getDeclaredField(key); - Class propertyType = Util.getParameterizedType(property.getGenericType()); - propertyType.getDeclaredField("additionalValuesJson"); + String dynamicFieldsName = "participantData".equals(key) ? "data" : "additionalValuesJson"; + propertyType.getDeclaredField(dynamicFieldsName); List> newObjects = new ArrayList<>(); for (Map object : objects) { Map clonedMap = new HashMap<>(object); @@ -104,77 +116,59 @@ private static ElasticSearchParticipantDto serialize(Map sourceM clonedMap.put("dynamicFields", dynamicFieldsValueAsJson); newObjects.add(clonedMap); } - clonedProperty.put(key, newObjects); + if (!newObjects.isEmpty()) + clonedProperty.put(key, newObjects); + } catch (NoSuchFieldException | ClassNotFoundException | JsonProcessingException e) { + if (((Map) dsm).containsKey("medicalRecords")) { + List> medicalRecords = (List>) ((Map) sourceMap.get(DSM)).get("medicalRecords"); + if (!Objects.isNull(medicalRecords)) { + for (Map medicalRecord : medicalRecords) { + if (StringUtils.isNotBlank((String) medicalRecord.get("followUps"))) { + String followUps = (String) medicalRecord.get("followUps"); + try { + medicalRecord.put("followUps", objectMapper.readValue(followUps, new TypeReference>>() { + })); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + } + } + elasticSearchParticipantDto = objectMapper.convertValue(sourceMap, ElasticSearchParticipantDto.class); + } else { + elasticSearchParticipantDto = objectMapper.convertValue(sourceMap, ElasticSearchParticipantDto.class); + } + } else { + elasticSearchParticipantDto = objectMapper.convertValue(sourceMap, ElasticSearchParticipantDto.class); + } + } + } else if("participant".equals(key) || "oncHistory".equals(key)) { + Map object = (Map) value; + try { + Field property = ESDsm.class.getDeclaredField(key); + Class propertyType = Util.getParameterizedType(property.getGenericType()); + String dynamicFieldsName = "additionalValuesJson"; + propertyType.getDeclaredField(dynamicFieldsName); + Map clonedObject = new HashMap<>(object); + Object dynamicFields = clonedObject.get("dynamicFields"); + String dynamicFieldsValueAsJson = Objects.isNull(dynamicFields) + ? "" + : objectMapper.writeValueAsString(dynamicFields); + clonedObject.put("dynamicFields", dynamicFieldsValueAsJson); + if(!clonedObject.isEmpty()) + clonedProperty.put(key, clonedObject); } catch (NoSuchFieldException | ClassNotFoundException | JsonProcessingException e) { e.printStackTrace(); } - - - } else { - } - dsmLevel.put(key, clonedProperty); } - - List> medicalRecords = (List>)((Map) sourceMap.get(DSM)).get("medicalRecords"); - if (!Objects.isNull(medicalRecords)) { - for (Map medicalRecord : medicalRecords) { - if (StringUtils.isNotBlank((String) medicalRecord.get("followUps"))) { - String followUps = (String) medicalRecord.get("followUps"); - try { - medicalRecord.put("followUps", objectMapper.readValue(followUps, new TypeReference>>(){})); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - } + if (!clonedProperty.isEmpty()) + dsmLevel.putAll(clonedProperty); -// if (hasFollowUps) { -// String sourceMapAsString = null; -// try { -// sourceMapAsString = new ObjectMapper().writeValueAsString(sourceMap); -// } catch (JsonProcessingException e) { -// throw new RuntimeException(e); -// } -// sourceMapAsString = sourceMapAsString.replace("\"[", "[").replace("]\"", "]").replace("\\\"", "\""); -// JsonReader reader = new JsonReader(new StringReader(sourceMapAsString)); -// reader.setLenient(true); -// elasticSearchParticipantDto = GSON.fromJson(reader, ElasticSearchParticipantDto.class); -// } else { -// elasticSearchParticipantDto = GSON.fromJson(GSON.toJson(sourceMap), ElasticSearchParticipantDto.class); -// } - elasticSearchParticipantDto = objectMapper.convertValue(sourceMap, ElasticSearchParticipantDto.class); - - } else { - elasticSearchParticipantDto = objectMapper.convertValue(sourceMap, ElasticSearchParticipantDto.class); - } } else { elasticSearchParticipantDto = objectMapper.convertValue(sourceMap, ElasticSearchParticipantDto.class); } return elasticSearchParticipantDto; -// boolean hasMedicalRecord = false; -// boolean hasDSM = sourceMap.containsKey(DSM); -// if (hasDSM) -// hasMedicalRecord = ((Map) sourceMap.get(DSM)).containsKey("medicalRecords"); -// long followUpsCount = 0; -// if (hasMedicalRecord) { -// followUpsCount = ((List>) (((Map) sourceMap.get(DSM)).get("medicalRecords"))).stream().filter(map -> map.containsKey("followUps")).count(); -// } -// if (followUpsCount > 0) { -// try { -// String sourceMapAsString = new ObjectMapper().writeValueAsString(sourceMap); -// sourceMapAsString = sourceMapAsString.replace("\"[", "[").replace("]\"", "]").replace("\\\"", "\""); -// JsonReader reader = new JsonReader(new StringReader(sourceMapAsString)); -// reader.setLenient(true); -// elasticSearchParticipantDto = GSON.fromJson(reader, ElasticSearchParticipantDto.class); -// } catch (JsonProcessingException e) { -// throw new RuntimeException(); -// } -// } else { -// elasticSearchParticipantDto = GSON.fromJson(GSON.toJson(sourceMap), ElasticSearchParticipantDto.class); -// } -// return elasticSearchParticipantDto; } private static String extractListFromString(String listInString) { diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java index b66b27e1a..967cf4286 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java @@ -92,9 +92,12 @@ public void getParameterizedType() throws NoSuchFieldException { class MockClass { List listField; FollowUp[] followUps; + Object obj; } + Field listField = MockClass.class.getDeclaredField("listField"); Field followUps = MockClass.class.getDeclaredField("followUps"); + Field obj = MockClass.class.getDeclaredField("obj"); Class clazz = null; try { @@ -102,6 +105,8 @@ class MockClass { assertEquals(Object.class, clazz); clazz = Util.getParameterizedType(followUps.getGenericType()); assertEquals(FollowUp.class, clazz); + clazz = Util.getParameterizedType(obj.getGenericType()); + assertEquals(Object.class, clazz); } catch (ClassNotFoundException e) { Assert.fail(); } From 59f82df33ae615d809669df62bd6eb9fb3e72dfa Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 2 Dec 2021 17:37:37 +0400 Subject: [PATCH 183/405] DDP-7121 | clean up serialize method --- .../model/elastic/search/ElasticSearch.java | 120 +++++++----------- 1 file changed, 44 insertions(+), 76 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java index 84c0f49aa..624f0ee5a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java @@ -78,7 +78,7 @@ public static Optional parseSourceMap(Map sourceMap) throws NoSuchFieldException, ClassNotFoundException { + private static ElasticSearchParticipantDto serialize(Map sourceMap) { ElasticSearchParticipantDto elasticSearchParticipantDto = null; Object dsm = sourceMap.get(DSM); ObjectMapper objectMapper = new ObjectMapper(); @@ -88,93 +88,61 @@ private static ElasticSearchParticipantDto serialize(Map sourceM for (Map.Entry entry: dsmLevel.entrySet()) { String key = entry.getKey(); //medicalRecords Object value = entry.getValue(); //medicalRecords: [] - Field property = ESDsm.class.getDeclaredField(key); - Class propertyType = Util.getParameterizedType(property.getGenericType()); - Field[] declaredFields = propertyType.getDeclaredFields(); - Optional maybeDynamicFields = Arrays.stream(declaredFields) - .filter(field -> field.getAnnotation(JsonProperty.class).value().equals("dynamicFields")) - .findFirst(); - if (hasDynamicFields) { - if (value instanceof List) { - - } else { - - } - } - if (value instanceof List) { - List> objects = (List>) value; - try { - String dynamicFieldsName = "participantData".equals(key) ? "data" : "additionalValuesJson"; - propertyType.getDeclaredField(dynamicFieldsName); - List> newObjects = new ArrayList<>(); - for (Map object : objects) { - Map clonedMap = new HashMap<>(object); - Object dynamicFields = clonedMap.get("dynamicFields"); + Field property = null; + try { + property = ESDsm.class.getDeclaredField(key); + Class propertyType = Util.getParameterizedType(property.getGenericType()); + Field[] declaredFields = propertyType.getDeclaredFields(); + boolean hasDynamicFields = Arrays.stream(declaredFields) + .anyMatch(field -> { + JsonProperty jsonProperty = field.getAnnotation(JsonProperty.class); + if (Objects.isNull(jsonProperty)) return false; + else return jsonProperty.value().equals("dynamicFields"); + }); + if (hasDynamicFields) { + if (value instanceof List) { + List> objects = (List>) value; + List> newObjects = new ArrayList<>(); + for (Map object : objects) { + Map clonedMap = new HashMap<>(object); + Object dynamicFields = clonedMap.get("dynamicFields"); + String followUps = (String)clonedMap.get("followUps"); + String dynamicFieldsValueAsJson = Objects.isNull(dynamicFields) + ? "" + : objectMapper.writeValueAsString(dynamicFields); + Object convertedFollowUps = Objects.isNull(followUps) + ? Collections.emptyList() + : objectMapper.readValue(followUps, new TypeReference>>() { + }); + clonedMap.put("dynamicFields", dynamicFieldsValueAsJson); + clonedMap.put("followUps", convertedFollowUps); + newObjects.add(clonedMap); + } + if (!newObjects.isEmpty()) + clonedProperty.put(key, newObjects); + } else { + Map object = (Map) value; + Map clonedObject = new HashMap<>(object); + Object dynamicFields = clonedObject.get("dynamicFields"); String dynamicFieldsValueAsJson = Objects.isNull(dynamicFields) ? "" : objectMapper.writeValueAsString(dynamicFields); - clonedMap.put("dynamicFields", dynamicFieldsValueAsJson); - newObjects.add(clonedMap); - } - if (!newObjects.isEmpty()) - clonedProperty.put(key, newObjects); - } catch (NoSuchFieldException | ClassNotFoundException | JsonProcessingException e) { - if (((Map) dsm).containsKey("medicalRecords")) { - List> medicalRecords = (List>) ((Map) sourceMap.get(DSM)).get("medicalRecords"); - if (!Objects.isNull(medicalRecords)) { - for (Map medicalRecord : medicalRecords) { - if (StringUtils.isNotBlank((String) medicalRecord.get("followUps"))) { - String followUps = (String) medicalRecord.get("followUps"); - try { - medicalRecord.put("followUps", objectMapper.readValue(followUps, new TypeReference>>() { - })); - } catch (IOException ioe) { - throw new RuntimeException(ioe); - } - } - } - elasticSearchParticipantDto = objectMapper.convertValue(sourceMap, ElasticSearchParticipantDto.class); - } else { - elasticSearchParticipantDto = objectMapper.convertValue(sourceMap, ElasticSearchParticipantDto.class); - } - } else { - elasticSearchParticipantDto = objectMapper.convertValue(sourceMap, ElasticSearchParticipantDto.class); - } - } - } else if("participant".equals(key) || "oncHistory".equals(key)) { - Map object = (Map) value; - try { - Field property = ESDsm.class.getDeclaredField(key); - Class propertyType = Util.getParameterizedType(property.getGenericType()); - String dynamicFieldsName = "additionalValuesJson"; - propertyType.getDeclaredField(dynamicFieldsName); - Map clonedObject = new HashMap<>(object); - Object dynamicFields = clonedObject.get("dynamicFields"); - String dynamicFieldsValueAsJson = Objects.isNull(dynamicFields) - ? "" - : objectMapper.writeValueAsString(dynamicFields); - clonedObject.put("dynamicFields", dynamicFieldsValueAsJson); - if(!clonedObject.isEmpty()) + clonedObject.put("dynamicFields", dynamicFieldsValueAsJson); clonedProperty.put(key, clonedObject); - } catch (NoSuchFieldException | ClassNotFoundException | JsonProcessingException e) { - e.printStackTrace(); + } } + } catch (NoSuchFieldException | ClassNotFoundException | IOException e) { + throw new RuntimeException(e); } } - if (!clonedProperty.isEmpty()) + if (!clonedProperty.isEmpty()) { dsmLevel.putAll(clonedProperty); - - - } else { - elasticSearchParticipantDto = objectMapper.convertValue(sourceMap, ElasticSearchParticipantDto.class); + } } + elasticSearchParticipantDto = objectMapper.convertValue(sourceMap, ElasticSearchParticipantDto.class); return elasticSearchParticipantDto; } - private static String extractListFromString(String listInString) { - return listInString.replace("\"[", "[").replace("]\"", "]").replace("\\\"", "\""); - } - public List parseSourceMaps(SearchHit[] searchHits) { if (Objects.isNull(searchHits)) return Collections.emptyList(); List result = new ArrayList<>(); From a3cdab2006a5911868e01e832184dc8e7b73fbd4 Mon Sep 17 00:00:00 2001 From: Nika Date: Fri, 3 Dec 2021 10:01:36 +0400 Subject: [PATCH 184/405] DDP-7121 add @JsonProperty and @SerializedName to some of the DTO fields, finish serialize() method in ElasticSearch.java --- .../broadinstitute/dsm/db/KitRequestShipping.java | 2 ++ .../org/broadinstitute/dsm/db/MedicalRecord.java | 15 +++++++++++++-- .../org/broadinstitute/dsm/db/OncHistory.java | 2 ++ .../broadinstitute/dsm/db/OncHistoryDetail.java | 9 +++++++++ .../org/broadinstitute/dsm/db/Participant.java | 9 +++++++++ .../broadinstitute/dsm/db/ParticipantData.java | 9 +++++++++ .../java/org/broadinstitute/dsm/db/Tissue.java | 9 +++++++++ .../broadinstitute/dsm/model/elastic/ESDsm.java | 9 ++------- .../broadinstitute/dsm/model/elastic/Util.java | 4 +++- 9 files changed, 58 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java index 9056de818..5a606d911 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java @@ -3,6 +3,7 @@ import com.easypost.exception.EasyPostException; import com.easypost.model.*; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; import lombok.Data; import lombok.NonNull; import org.apache.commons.lang3.StringUtils; @@ -41,6 +42,7 @@ primaryKey = DBConstants.DSM_KIT_REQUEST_ID, columnPrefix = "") @JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) public class KitRequestShipping extends KitRequest { private static final Logger logger = LoggerFactory.getLogger(KitRequestShipping.class); diff --git a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java index c8ca2b7b6..70e45ed11 100644 --- a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java +++ b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java @@ -12,8 +12,12 @@ import java.util.Map; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; +import com.google.gson.annotations.SerializedName; import lombok.Data; import lombok.NonNull; import org.broadinstitute.ddp.db.SimpleResult; @@ -39,6 +43,7 @@ columnPrefix = "") @Data @JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) public class MedicalRecord { private static final Logger logger = LoggerFactory.getLogger(MedicalRecord.class); @@ -332,11 +337,17 @@ public boolean isFollowUpRequired() { primaryKey = DBConstants.MEDICAL_RECORD_ID, columnPrefix = "") @ColumnName (DBConstants.ADDITIONAL_VALUES) + @JsonProperty("dynamicFields") + @SerializedName("dynamicFields") private String additionalValuesJson; @JsonProperty("dynamicFields") - public String getAdditionalValuesJson() { - return additionalValuesJson; + public Map getAdditionalValuesJson() { + try { + return new ObjectMapper().readValue(additionalValuesJson, new TypeReference>() {}); + } catch (IOException e) { + return Map.of(); + } } private boolean reviewMedicalRecord; diff --git a/src/main/java/org/broadinstitute/dsm/db/OncHistory.java b/src/main/java/org/broadinstitute/dsm/db/OncHistory.java index bf6a9adf7..0ae50b58b 100644 --- a/src/main/java/org/broadinstitute/dsm/db/OncHistory.java +++ b/src/main/java/org/broadinstitute/dsm/db/OncHistory.java @@ -1,5 +1,6 @@ package org.broadinstitute.dsm.db; +import com.fasterxml.jackson.annotation.JsonInclude; import lombok.NonNull; import org.broadinstitute.ddp.db.SimpleResult; import org.broadinstitute.dsm.model.NameValue; @@ -12,6 +13,7 @@ import static org.broadinstitute.ddp.db.TransactionWrapper.inTransaction; +@JsonInclude(JsonInclude.Include.NON_NULL) public class OncHistory { private static final Logger logger = LoggerFactory.getLogger(OncHistory.class); diff --git a/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java b/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java index 8657b1bde..933c0a08e 100644 --- a/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java +++ b/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java @@ -1,6 +1,9 @@ package org.broadinstitute.dsm.db; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.gson.annotations.SerializedName; import lombok.Data; import lombok.NonNull; import org.apache.commons.lang3.StringUtils; @@ -31,6 +34,7 @@ primaryKey = DBConstants.ONC_HISTORY_DETAIL_ID, columnPrefix = "") @JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) public class OncHistoryDetail { private static final Logger logger = LoggerFactory.getLogger(OncHistoryDetail.class); @@ -158,8 +162,13 @@ public class OncHistoryDetail { private String gender; @ColumnName (DBConstants.ADDITIONAL_VALUES) + @JsonProperty("dynamicFields") + @SerializedName("dynamicFields") private String additionalValuesJson; + @JsonProperty("dynamicFields") + public String getAdditionalValuesJson() {return additionalValuesJson;} + @ColumnName (DBConstants.DESTRUCTION_POLICY) private String destructionPolicy; diff --git a/src/main/java/org/broadinstitute/dsm/db/Participant.java b/src/main/java/org/broadinstitute/dsm/db/Participant.java index d02ecbd51..a2de4733f 100644 --- a/src/main/java/org/broadinstitute/dsm/db/Participant.java +++ b/src/main/java/org/broadinstitute/dsm/db/Participant.java @@ -1,6 +1,9 @@ package org.broadinstitute.dsm.db; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.gson.annotations.SerializedName; import lombok.Data; import lombok.NonNull; import org.apache.commons.lang3.StringUtils; @@ -21,6 +24,7 @@ @Data @JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) public class Participant { private static final Logger logger = LoggerFactory.getLogger(Participant.class); @@ -120,8 +124,13 @@ public class Participant { primaryKey = DBConstants.PARTICIPANT_ID, columnPrefix = "") @ColumnName (DBConstants.ADDITIONAL_VALUES) + @JsonProperty("dynamicFields") + @SerializedName("dynamicFields") private String additionalValuesJson; + @JsonProperty("dynamicFields") + public String getAdditionalValuesJson() { return additionalValuesJson; } + @TableName ( name = DBConstants.DDP_PARTICIPANT_EXIT, alias = DBConstants.DDP_PARTICIPANT_EXIT_ALIAS, diff --git a/src/main/java/org/broadinstitute/dsm/db/ParticipantData.java b/src/main/java/org/broadinstitute/dsm/db/ParticipantData.java index 4889aec86..38e0a6990 100644 --- a/src/main/java/org/broadinstitute/dsm/db/ParticipantData.java +++ b/src/main/java/org/broadinstitute/dsm/db/ParticipantData.java @@ -1,5 +1,8 @@ package org.broadinstitute.dsm.db; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.gson.annotations.SerializedName; import lombok.Data; import lombok.NonNull; import org.broadinstitute.ddp.db.SimpleResult; @@ -27,6 +30,7 @@ alias = DBConstants.DDP_PARTICIPANT_DATA_ALIAS, primaryKey = DBConstants.PARTICIPANT_DATA_ID, columnPrefix = "") +@JsonInclude(JsonInclude.Include.NON_NULL) public class ParticipantData { private static final Logger logger = LoggerFactory.getLogger(ParticipantData.class); @@ -46,8 +50,13 @@ public class ParticipantData { private final String fieldTypeId; @ColumnName (DBConstants.DATA) + @JsonProperty("dynamicFields") + @SerializedName("dynamicFields") private final String data; + @JsonProperty("dynamicFields") + public String getData() {return data;} + public ParticipantData(String dataId, String fieldTypeId, String data) { this.dataId = dataId; this.fieldTypeId = fieldTypeId; diff --git a/src/main/java/org/broadinstitute/dsm/db/Tissue.java b/src/main/java/org/broadinstitute/dsm/db/Tissue.java index bf156dc1f..98f7de17f 100644 --- a/src/main/java/org/broadinstitute/dsm/db/Tissue.java +++ b/src/main/java/org/broadinstitute/dsm/db/Tissue.java @@ -1,5 +1,8 @@ package org.broadinstitute.dsm.db; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.gson.annotations.SerializedName; import lombok.Data; import lombok.NonNull; import org.apache.commons.lang3.StringUtils; @@ -24,6 +27,7 @@ alias = DBConstants.DDP_TISSUE_ALIAS, primaryKey = DBConstants.TISSUE_ID, columnPrefix = "") +@JsonInclude(JsonInclude.Include.NON_NULL) public class Tissue { private static final Logger logger = LoggerFactory.getLogger(Tissue.class); @@ -102,8 +106,13 @@ public class Tissue { private String firstSmId; @ColumnName (DBConstants.ADDITIONAL_TISSUE_VALUES) + @JsonProperty("dynamicFields") + @SerializedName("dynamicFields") private String additionalValuesJson; + @JsonProperty("dynamicFields") + public String getAdditionalValuesJson() {return additionalValuesJson;} + @ColumnName (DBConstants.TISSUE_RETURN_DATE) @DbDateConversion(SqlDateConverter.STRING_DAY) private String returnDate; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java index 50c0ad00d..99f6344bc 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java @@ -5,12 +5,7 @@ import com.google.gson.annotations.SerializedName; import lombok.Getter; -import org.broadinstitute.dsm.db.KitRequestShipping; -import org.broadinstitute.dsm.db.MedicalRecord; -import org.broadinstitute.dsm.db.OncHistory; -import org.broadinstitute.dsm.db.OncHistoryDetail; -import org.broadinstitute.dsm.db.Participant; -import org.broadinstitute.dsm.db.ParticipantData; +import org.broadinstitute.dsm.db.*; import org.broadinstitute.dsm.statics.ESObjectConstants; @Getter @@ -41,7 +36,7 @@ public class ESDsm { Object diagnosisYear; @SerializedName(ESObjectConstants.TISSUE_RECORDS) - List> tissueRecords; // todo: change to Tissue + List tissueRecords; // todo: change to Tissue @SerializedName(ESObjectConstants.MEDICAL_RECORDS) List medicalRecords; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index ea41efc66..d500940b6 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -7,6 +7,7 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.api.client.json.Json; import com.google.common.reflect.TypeToken; @@ -136,7 +137,8 @@ static Map convertToMap(String fieldName, Object fieldValue) { } public static List> convertObjectListToMapList(Object fieldValue) { - return GSON.fromJson(GSON.toJson(fieldValue), new TypeToken>>(){}.getType()); + return new ObjectMapper().convertValue(fieldValue, new TypeReference>>() {}); +// return GSON.fromJson(GSON.toJson(fieldValue), new TypeToken>>(){}.getType()); } private static Map dynamicFieldsSpecialCase(Object fieldValue) { From 06350e29b79d53afbe6350c814eaa34fbada2eca Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 3 Dec 2021 10:19:13 +0400 Subject: [PATCH 185/405] DDP-7121 | change getter method name --- .../org/broadinstitute/dsm/db/MedicalRecord.java | 2 +- .../org/broadinstitute/dsm/db/OncHistoryDetail.java | 11 ++++++++++- .../java/org/broadinstitute/dsm/db/Participant.java | 11 ++++++++++- .../org/broadinstitute/dsm/db/ParticipantData.java | 12 ++++++++++++ src/main/java/org/broadinstitute/dsm/db/Tissue.java | 12 +++++++++++- 5 files changed, 44 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java index 70e45ed11..e0260e178 100644 --- a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java +++ b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java @@ -342,7 +342,7 @@ public boolean isFollowUpRequired() { private String additionalValuesJson; @JsonProperty("dynamicFields") - public Map getAdditionalValuesJson() { + public Map getDynamicFields() { try { return new ObjectMapper().readValue(additionalValuesJson, new TypeReference>() {}); } catch (IOException e) { diff --git a/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java b/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java index 933c0a08e..0c883afab 100644 --- a/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java +++ b/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java @@ -3,6 +3,8 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.annotations.SerializedName; import lombok.Data; import lombok.NonNull; @@ -19,6 +21,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; @@ -167,7 +170,13 @@ public class OncHistoryDetail { private String additionalValuesJson; @JsonProperty("dynamicFields") - public String getAdditionalValuesJson() {return additionalValuesJson;} + public Map getDynamicFields() { + try { + return new ObjectMapper().readValue(additionalValuesJson, new TypeReference>() {}); + } catch (IOException e) { + return Map.of(); + } + } @ColumnName (DBConstants.DESTRUCTION_POLICY) private String destructionPolicy; diff --git a/src/main/java/org/broadinstitute/dsm/db/Participant.java b/src/main/java/org/broadinstitute/dsm/db/Participant.java index a2de4733f..c11c4da45 100644 --- a/src/main/java/org/broadinstitute/dsm/db/Participant.java +++ b/src/main/java/org/broadinstitute/dsm/db/Participant.java @@ -3,6 +3,8 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.annotations.SerializedName; import lombok.Data; import lombok.NonNull; @@ -15,6 +17,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; @@ -129,7 +132,13 @@ public class Participant { private String additionalValuesJson; @JsonProperty("dynamicFields") - public String getAdditionalValuesJson() { return additionalValuesJson; } + public Map getDynamicFields() { + try { + return new ObjectMapper().readValue(additionalValuesJson, new TypeReference>() {}); + } catch (IOException e) { + return Map.of(); + } + } @TableName ( name = DBConstants.DDP_PARTICIPANT_EXIT, diff --git a/src/main/java/org/broadinstitute/dsm/db/ParticipantData.java b/src/main/java/org/broadinstitute/dsm/db/ParticipantData.java index 38e0a6990..ab01187f5 100644 --- a/src/main/java/org/broadinstitute/dsm/db/ParticipantData.java +++ b/src/main/java/org/broadinstitute/dsm/db/ParticipantData.java @@ -2,6 +2,8 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.annotations.SerializedName; import lombok.Data; import lombok.NonNull; @@ -13,6 +15,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; @@ -57,6 +60,15 @@ public class ParticipantData { @JsonProperty("dynamicFields") public String getData() {return data;} + @JsonProperty("dynamicFields") + public Map getDynamicFields() { + try { + return new ObjectMapper().readValue(data, new TypeReference>() {}); + } catch (IOException e) { + return Map.of(); + } + } + public ParticipantData(String dataId, String fieldTypeId, String data) { this.dataId = dataId; this.fieldTypeId = fieldTypeId; diff --git a/src/main/java/org/broadinstitute/dsm/db/Tissue.java b/src/main/java/org/broadinstitute/dsm/db/Tissue.java index 98f7de17f..91c9b814e 100644 --- a/src/main/java/org/broadinstitute/dsm/db/Tissue.java +++ b/src/main/java/org/broadinstitute/dsm/db/Tissue.java @@ -2,6 +2,8 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.annotations.SerializedName; import lombok.Data; import lombok.NonNull; @@ -15,9 +17,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.sql.*; import java.util.ArrayList; import java.util.List; +import java.util.Map; import static org.broadinstitute.ddp.db.TransactionWrapper.inTransaction; @@ -111,7 +115,13 @@ public class Tissue { private String additionalValuesJson; @JsonProperty("dynamicFields") - public String getAdditionalValuesJson() {return additionalValuesJson;} + public Map getDynamicFields() { + try { + return new ObjectMapper().readValue(additionalValuesJson, new TypeReference>() {}); + } catch (IOException e) { + return Map.of(); + } + } @ColumnName (DBConstants.TISSUE_RETURN_DATE) @DbDateConversion(SqlDateConverter.STRING_DAY) From 09d92335bd4e5638d8c970935c0517e9c6080d8a Mon Sep 17 00:00:00 2001 From: Nika Date: Fri, 3 Dec 2021 11:01:21 +0400 Subject: [PATCH 186/405] DDP-7121 | fix updateExistingRecord() in CollectionProcessor.java --- .../export/process/CollectionProcessor.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java index fa8bfe03f..8d5f094ef 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java @@ -89,7 +89,20 @@ private boolean isExistingRecord(Map eachRecord) { private void updateExistingRecord(Map eachRecord) { logger.info("Updating existing record"); - collectEndResult().ifPresent(eachRecord::putAll); + Optional> maybeEndResult = collectEndResult(); + if (maybeEndResult.isPresent()) { + for (Map.Entry endResultEntry : maybeEndResult.get().entrySet()) { + String endResultEntryKey = endResultEntry.getKey(); + Object eachRecordValue = eachRecord.get(endResultEntryKey); + if (eachRecordValue instanceof Map) { + Map eachRecordEntryMap = (Map) eachRecordValue; + eachRecordEntryMap.putAll((Map) endResultEntry.getValue()); + } else { + eachRecord.putAll(maybeEndResult.get()); + } + } + } + } private void addNewRecordTo(List> fetchedRecords) { From 3252c89873ce7b96d9cacbd3c44a5b98e70372a5 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 3 Dec 2021 11:37:43 +0400 Subject: [PATCH 187/405] DDP-7121 | add json property for follow ups to avoid duplication, remove commented out code --- src/main/java/org/broadinstitute/dsm/model/FollowUp.java | 5 +++++ src/main/java/org/broadinstitute/dsm/model/elastic/Util.java | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/FollowUp.java b/src/main/java/org/broadinstitute/dsm/model/FollowUp.java index 28b0ed831..35b466c4b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/FollowUp.java +++ b/src/main/java/org/broadinstitute/dsm/model/FollowUp.java @@ -1,5 +1,6 @@ package org.broadinstitute.dsm.model; +import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; import org.broadinstitute.dsm.db.structure.ColumnName; @@ -7,15 +8,19 @@ public class FollowUp { @ColumnName("fReceived") + @JsonProperty("fReceived") public String fReceived; @ColumnName("fRequest1") + @JsonProperty("fRequest1") public String fRequest1; @ColumnName("fRequest2") + @JsonProperty("fRequest2") public String fRequest2; @ColumnName("fRequest3") + @JsonProperty("fRequest3") public String fRequest3; public FollowUp(String fRequest1, String fRequest2, String fRequest3, String fReceived) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index d500940b6..930fad950 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -138,7 +138,6 @@ static Map convertToMap(String fieldName, Object fieldValue) { public static List> convertObjectListToMapList(Object fieldValue) { return new ObjectMapper().convertValue(fieldValue, new TypeReference>>() {}); -// return GSON.fromJson(GSON.toJson(fieldValue), new TypeToken>>(){}.getType()); } private static Map dynamicFieldsSpecialCase(Object fieldValue) { From 3a1ed82526136aa10f1e5867bb49ca3130d82666 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 3 Dec 2021 16:18:03 +0400 Subject: [PATCH 188/405] DDP-7121 | remove dead code --- .../java/org/broadinstitute/dsm/model/elastic/ESDsm.java | 2 ++ .../model/elastic/export/generate/MappingGenerator.java | 5 ----- .../elastic/export/generate/SingleMappingGenerator.java | 9 --------- 3 files changed, 2 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java index 99f6344bc..3c680d78a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java @@ -6,6 +6,7 @@ import com.google.gson.annotations.SerializedName; import lombok.Getter; import org.broadinstitute.dsm.db.*; +import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantRecordDto; import org.broadinstitute.dsm.statics.ESObjectConstants; @Getter @@ -53,4 +54,5 @@ public class ESDsm { List kitRequestShipping; OncHistory oncHistory; + } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java index 30075f126..2a256aa49 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java @@ -16,7 +16,6 @@ abstract public class MappingGenerator extends BaseGenerator { public static final String TYPE = "type"; public static final String NESTED = "nested"; public static final String TYPE_KEYWORD = "keyword"; - protected Generator dynamicFieldsMappingGenerator; public MappingGenerator(Parser parser, GeneratorPayload generatorPayload) { super(parser, generatorPayload); @@ -24,10 +23,6 @@ public MappingGenerator(Parser parser, GeneratorPayload generatorPayload) { public MappingGenerator() {} - public MappingGenerator(Generator dynamicFieldsMappingGenerator) { - this.dynamicFieldsMappingGenerator = dynamicFieldsMappingGenerator; - } - @Override public Map generate() { logger.info("preparing mapping to upsert"); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java index 7c931dea6..a96dc7525 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java @@ -17,17 +17,8 @@ public SingleMappingGenerator(Parser parser, GeneratorPayload generatorPayload) public SingleMappingGenerator() {} - public SingleMappingGenerator(Generator dynamicFieldsMappingGenerator) { - super(dynamicFieldsMappingGenerator); - } - @Override protected Map getElement(Object type) { - if (!Objects.nonNull(dynamicFieldsMappingGenerator)) { - dynamicFieldsMappingGenerator.setPayload(generatorPayload); - dynamicFieldsMappingGenerator.setParser(parser); - return dynamicFieldsMappingGenerator.generate(); - } return new HashMap<>(Map.of(getFieldName(), type)); } From 0cd74c409266210d276667f9669007cbfcd07dcc Mon Sep 17 00:00:00 2001 From: Nika Date: Fri, 3 Dec 2021 16:50:28 +0400 Subject: [PATCH 189/405] DDP-7121 | refactor ParticipantRecordDto --- .../ddp/participant/ParticipantRecordDto.java | 139 ++++++++++++++---- .../dsm/model/elastic/Util.java | 2 +- .../dsm/statics/DBConstants.java | 1 + .../broadinstitute/dsm/util/PatchUtil.java | 2 + 4 files changed, 118 insertions(+), 26 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantRecordDto.java b/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantRecordDto.java index a4f5c91d0..89eec91cb 100644 --- a/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantRecordDto.java +++ b/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantRecordDto.java @@ -1,22 +1,111 @@ package org.broadinstitute.dsm.db.dto.ddp.participant; +import java.io.IOException; +import java.util.Map; import java.util.Optional; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.gson.annotations.SerializedName; +import lombok.Data; import lombok.Getter; +import org.broadinstitute.dsm.db.structure.ColumnName; +import org.broadinstitute.dsm.db.structure.TableName; +import org.broadinstitute.dsm.statics.DBConstants; -@Getter +@TableName( + name = DBConstants.DDP_PARTICIPANT_RECORD, + alias = DBConstants.DDP_PARTICIPANT_RECORD_ALIAS, + primaryKey = DBConstants.PARTICIPANT_RECORD_ID, + columnPrefix = "") +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) public class ParticipantRecordDto { - private Optional participantRecordId; + @ColumnName(DBConstants.PARTICIPANT_RECORD_ID) + private Integer participantRecordId; + + @ColumnName (DBConstants.PARTICIPANT_ID) private int participantId; - private Optional crSent; - private Optional crReceived; - private Optional notes; - private Optional minimalMr; - private Optional abstractionReady; - private Optional additionalValuesJson; + + @ColumnName (DBConstants.CR_SENT) + private String crSent; + + @ColumnName (DBConstants.CR_RECEIVED) + private String crReceived; + + @ColumnName (DBConstants.NOTES) + private String notes; + + @ColumnName (DBConstants.MINIMAL_MR) + private Integer minimalMr; + + @ColumnName (DBConstants.ABSTRACTION_READY) + private Integer abstractionReady; + + @ColumnName (DBConstants.ADDITIONAL_VALUES) + @JsonProperty("dynamicFields") + private String additionalValuesJson; + + @JsonProperty("dynamicFields") + public Map getDynamicFields() { + try { + return new ObjectMapper().readValue(additionalValuesJson, new TypeReference>() {}); + } catch (IOException e) { + return Map.of(); + } + } + + private long lastChanged; - private Optional changedBy; + private String changedBy; + + public ParticipantRecordDto() { + + } + + public Optional getParticipantRecordId() { + return Optional.ofNullable(participantRecordId); + } + + public int getParticipantId() { + return participantId; + } + + public Optional getCrSent() { + return Optional.ofNullable(crSent); + } + + public Optional getCrReceived() { + return Optional.ofNullable(crReceived); + } + + public Optional getNotes() { + return Optional.ofNullable(notes); + } + + public Optional getMinimalMr() { + return Optional.ofNullable(minimalMr); + } + + public Optional getAbstractionReady() { + return Optional.ofNullable(abstractionReady); + } + + public Optional getAdditionalValuesJson() { + return Optional.ofNullable(additionalValuesJson); + } + + public long getLastChanged() { + return lastChanged; + } + + public Optional getChangedBy() { + return Optional.ofNullable(changedBy); + } private ParticipantRecordDto(Builder builder) { this.participantRecordId = builder.participantRecordId; @@ -32,16 +121,16 @@ private ParticipantRecordDto(Builder builder) { } public static class Builder { - public Optional participantRecordId = Optional.empty(); + public Integer participantRecordId; public int participantId; - public Optional crSent = Optional.empty(); - public Optional crReceived = Optional.empty(); - public Optional notes = Optional.empty(); - public Optional minimalMr = Optional.empty(); - public Optional abstractionReady = Optional.empty(); - public Optional additionalValuesJson = Optional.empty(); + public String crSent; + public String crReceived; + public String notes; + public Integer minimalMr; + public Integer abstractionReady; + public String additionalValuesJson; public long lastChanged; - public Optional changedBy = Optional.empty(); + public String changedBy; public Builder(int participantId, long lastChanged) { this.participantId = participantId; @@ -49,42 +138,42 @@ public Builder(int participantId, long lastChanged) { } public Builder withParticipantRecordId(int participantRecordId) { - this.participantRecordId = Optional.of(participantRecordId); + this.participantRecordId = participantRecordId; return this; } public Builder withCrSent(String crSent) { - this.crSent = Optional.ofNullable(crSent); + this.crSent = crSent; return this; } public Builder withCrReceived(String crReceived) { - this.crReceived = Optional.ofNullable(crReceived); + this.crReceived = crReceived; return this; } public Builder withNotes(String notes) { - this.notes = Optional.ofNullable(notes); + this.notes = notes; return this; } public Builder withMinimalMr(int minimalMr) { - this.minimalMr = Optional.of(minimalMr); + this.minimalMr = minimalMr; return this; } public Builder withAbstractionReady(int abstractionReady) { - this.abstractionReady = Optional.of(abstractionReady); + this.abstractionReady = abstractionReady; return this; } public Builder withAdditionalValuesJson(String additionalValuesJson) { - this.additionalValuesJson = Optional.ofNullable(additionalValuesJson); + this.additionalValuesJson = additionalValuesJson; return this; } public Builder withChangedBy(String changedBy) { - this.changedBy = Optional.ofNullable(changedBy); + this.changedBy = changedBy; return this; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 930fad950..4b5431a85 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -27,7 +27,7 @@ public class Util { "t", new BaseGenerator.PropertyInfo(ESObjectConstants.TISSUE_RECORDS, true), "oD", new BaseGenerator.PropertyInfo(ESObjectConstants.ONC_HISTORY_DETAIL_RECORDS, true), "d", new BaseGenerator.PropertyInfo(ESObjectConstants.PARTICIPANT_DATA, true), - "r", new BaseGenerator.PropertyInfo(ESObjectConstants.PARTICIPANT_RECORD, false), + "r", new BaseGenerator.PropertyInfo(ESObjectConstants.PARTICIPANT, false), "p", new BaseGenerator.PropertyInfo(ESObjectConstants.PARTICIPANT, false), "o", new BaseGenerator.PropertyInfo(ESObjectConstants.ONC_HISTORY, false) ); diff --git a/src/main/java/org/broadinstitute/dsm/statics/DBConstants.java b/src/main/java/org/broadinstitute/dsm/statics/DBConstants.java index 64034f7af..66659716f 100644 --- a/src/main/java/org/broadinstitute/dsm/statics/DBConstants.java +++ b/src/main/java/org/broadinstitute/dsm/statics/DBConstants.java @@ -487,4 +487,5 @@ public class DBConstants { public static final String LABEL_URL_RETURN = "label_url_return"; + public static final String PARTICIPANT_RECORD_ID = "participant_record_id"; } diff --git a/src/main/java/org/broadinstitute/dsm/util/PatchUtil.java b/src/main/java/org/broadinstitute/dsm/util/PatchUtil.java index 08cd43afa..63259b2c4 100644 --- a/src/main/java/org/broadinstitute/dsm/util/PatchUtil.java +++ b/src/main/java/org/broadinstitute/dsm/util/PatchUtil.java @@ -2,6 +2,7 @@ import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.*; +import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantRecordDto; import org.broadinstitute.dsm.db.structure.ColumnName; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.db.structure.DbDateConversion; @@ -41,6 +42,7 @@ public class PatchUtil { getColumnNames(KitRequest.class); getColumnNames(Drug.class); getColumnNames(ParticipantData.class); + getColumnNames(ParticipantRecordDto.class); logger.info("Loaded patch utils"); } From baf27d2ef8dcd9939b082add94b831f890380b8d Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 3 Dec 2021 17:20:29 +0400 Subject: [PATCH 190/405] DDP-7121 | add empty constructor of onc history for jackson --- .../java/org/broadinstitute/dsm/db/OncHistory.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/OncHistory.java b/src/main/java/org/broadinstitute/dsm/db/OncHistory.java index 0ae50b58b..c1956512b 100644 --- a/src/main/java/org/broadinstitute/dsm/db/OncHistory.java +++ b/src/main/java/org/broadinstitute/dsm/db/OncHistory.java @@ -20,10 +20,12 @@ public class OncHistory { private static final String SQL_UPDATE_ONC_HISTORY = "UPDATE ddp_onc_history SET created = ?, last_changed = ?, changed_by = ? WHERE participant_id = ? AND created IS NULL"; - private final String participantId; - private final String created; - private final String reviewed; - private final String changedBy; + private String participantId; + private String created; + private String reviewed; + private String changedBy; + + public OncHistory() {} public OncHistory (String participantId, String created, String reviewed, String changedBy) { this.participantId = participantId; From 7dc73c04be84732add850ace812bdf227b0891f0 Mon Sep 17 00:00:00 2001 From: Nika Date: Fri, 3 Dec 2021 17:45:06 +0400 Subject: [PATCH 191/405] DDP-7121 | rename oncHistoryDetailRecords -> oncHistoryDetails --- src/main/java/org/broadinstitute/dsm/model/elastic/Util.java | 2 +- .../java/org/broadinstitute/dsm/statics/ESObjectConstants.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 4b5431a85..5f4a02986 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -25,7 +25,7 @@ public class Util { public static final Map TABLE_ALIAS_MAPPINGS = Map.of( "m", new BaseGenerator.PropertyInfo(ESObjectConstants.MEDICAL_RECORDS, true), "t", new BaseGenerator.PropertyInfo(ESObjectConstants.TISSUE_RECORDS, true), - "oD", new BaseGenerator.PropertyInfo(ESObjectConstants.ONC_HISTORY_DETAIL_RECORDS, true), + "oD", new BaseGenerator.PropertyInfo(ESObjectConstants.ONC_HISTORY_DETAILS, true), "d", new BaseGenerator.PropertyInfo(ESObjectConstants.PARTICIPANT_DATA, true), "r", new BaseGenerator.PropertyInfo(ESObjectConstants.PARTICIPANT, false), "p", new BaseGenerator.PropertyInfo(ESObjectConstants.PARTICIPANT, false), diff --git a/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java b/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java index e4bbf03da..0b4e80897 100644 --- a/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java +++ b/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java @@ -66,7 +66,7 @@ public class ESObjectConstants { public static final String DIAGNOSIS_MONTH = "diagnosisMonth"; public static final String HAS_CONSENTED_TO_TISSUE_SAMPLE = "hasConsentedToTissueSample"; public static final String DIAGNOSIS_YEAR = "diagnosisYear"; - public static final String ONC_HISTORY_DETAIL_RECORDS = "oncHistoryDetailRecords"; + public static final String ONC_HISTORY_DETAIL_RECORDS = "oncHistoryDetails"; public static final String ONC_HISTORY_DETAILS = "oncHistoryDetails"; public static final String ONC_HISTORY = "oncHistory"; public static final String PARTICIPANT_DATA = "participantData"; From 8e6730b318c546c2a760dffd4cbb061ea5ef6952 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 3 Dec 2021 19:19:36 +0400 Subject: [PATCH 192/405] DDP-7121 | change property names to classes --- .../dsm/model/elastic/Util.java | 20 +++++++++++------- .../export/generate/BaseGenerator.java | 21 ++++++++++++++----- .../generate/CollectionSourceGenerator.java | 10 +++++++-- .../DynamicFieldsMappingMigrator.java | 3 ++- 4 files changed, 39 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 5f4a02986..16fda02ff 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -13,6 +13,12 @@ import com.google.common.reflect.TypeToken; import com.google.gson.Gson; import org.apache.commons.lang3.StringUtils; +import org.broadinstitute.dsm.db.MedicalRecord; +import org.broadinstitute.dsm.db.OncHistory; +import org.broadinstitute.dsm.db.OncHistoryDetail; +import org.broadinstitute.dsm.db.Participant; +import org.broadinstitute.dsm.db.ParticipantData; +import org.broadinstitute.dsm.db.Tissue; import org.broadinstitute.dsm.db.structure.ColumnName; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; @@ -23,13 +29,13 @@ public class Util { public static final Map TABLE_ALIAS_MAPPINGS = Map.of( - "m", new BaseGenerator.PropertyInfo(ESObjectConstants.MEDICAL_RECORDS, true), - "t", new BaseGenerator.PropertyInfo(ESObjectConstants.TISSUE_RECORDS, true), - "oD", new BaseGenerator.PropertyInfo(ESObjectConstants.ONC_HISTORY_DETAILS, true), - "d", new BaseGenerator.PropertyInfo(ESObjectConstants.PARTICIPANT_DATA, true), - "r", new BaseGenerator.PropertyInfo(ESObjectConstants.PARTICIPANT, false), - "p", new BaseGenerator.PropertyInfo(ESObjectConstants.PARTICIPANT, false), - "o", new BaseGenerator.PropertyInfo(ESObjectConstants.ONC_HISTORY, false) + "m", new BaseGenerator.PropertyInfo(MedicalRecord.class, true), + "t", new BaseGenerator.PropertyInfo(Tissue.class, true), + "oD", new BaseGenerator.PropertyInfo(OncHistoryDetail.class, true), + "d", new BaseGenerator.PropertyInfo(ParticipantData.class, true), + "r", new BaseGenerator.PropertyInfo(Participant.class, false), + "p", new BaseGenerator.PropertyInfo(Participant.class, false), + "o", new BaseGenerator.PropertyInfo(OncHistory.class, false) ); public static final int FIRST_ELEMENT_INDEX = 0; public static final String UNDERSCORE_SEPARATOR = "_"; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java index 1313b5312..084e6b07a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java @@ -1,6 +1,5 @@ package org.broadinstitute.dsm.model.elastic.export.generate; -import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -9,6 +8,7 @@ import com.google.gson.JsonSyntaxException; import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.structure.DBElement; +import org.broadinstitute.dsm.db.structure.TableName; import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; @@ -66,6 +66,10 @@ private PropertyInfo getOuterPropertyByAlias() { return Util.TABLE_ALIAS_MAPPINGS.get(getDBElement().getTableAlias()); } + protected String getPrimaryKey() { + return getOuterPropertyByAlias().getPrimaryKey(); + } + public String getPropertyName() { return getOuterPropertyByAlias().getPropertyName(); } @@ -107,12 +111,12 @@ public Map merge(Map base, Map t public static class PropertyInfo { - private String propertyName; + private Class propertyClass; private boolean isCollection; private String fieldName; - public PropertyInfo(String propertyName, boolean isCollection) { - this.propertyName = Objects.requireNonNull(propertyName); + public PropertyInfo(Class propertyClass, boolean isCollection) { + this.propertyClass = Objects.requireNonNull(propertyClass); this.isCollection = isCollection; } @@ -121,7 +125,14 @@ public void setIsCollection(boolean isCollection) { } public String getPropertyName() { - return propertyName; + StringBuilder className = new StringBuilder(propertyClass.getSimpleName()); + StringBuilder camelCaseClassName = className.replace(0, 1, String.valueOf(className.charAt(0)).toLowerCase()); + return camelCaseClassName.toString(); + } + + public String getPrimaryKey() { + TableName tableName = Objects.requireNonNull(propertyClass.getAnnotation(TableName.class)); + return Util.underscoresToCamelCase(tableName.primaryKey()); } public boolean isCollection() { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java index 072a67779..2bac83c66 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java @@ -25,12 +25,18 @@ public CollectionSourceGenerator() { @Override public Object construct() { logger.info("Constructing nested data"); - Map mapWithParsedObjects = new HashMap<>(Map.of("dynamicFields", parseJsonValuesToObject())); + Map mapWithParsedObjects = new HashMap<>(Map.of( + getPrimaryKey(), generatorPayload.getRecordId(), + "dynamicFields", parseJsonValuesToObject()) + ); return List.of(mapWithParsedObjects); } @Override protected Object getElement(Object element) { - return List.of(new HashMap<>(Map.of(getFieldName(), element))); + return List.of(new HashMap<>(Map.of( + getPrimaryKey(), generatorPayload.getRecordId(), + getFieldName(), element) + )); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java index 683368eca..3a03ad287 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java @@ -7,6 +7,7 @@ import java.util.List; import java.util.Map; +import org.broadinstitute.dsm.db.ParticipantData; import org.broadinstitute.dsm.db.dao.settings.FieldSettingsDao; import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; import org.broadinstitute.dsm.model.elastic.Util; @@ -48,7 +49,7 @@ public void export() { if (propertyInfo != null) buildMapping(fieldSettingsDto, propertyInfo); else - buildMapping(fieldSettingsDto, new BaseGenerator.PropertyInfo(ESObjectConstants.PARTICIPANT_DATA, true)); + buildMapping(fieldSettingsDto, new BaseGenerator.PropertyInfo(ParticipantData.class, true)); } elasticMappingExportAdapter.setRequestPayload(new RequestPayload(index)); elasticMappingExportAdapter.setSource(buildFinalMapping()); From 31e843082dc4f4e3cca63894adabcc0b168dba5f Mon Sep 17 00:00:00 2001 From: Nika Date: Fri, 3 Dec 2021 19:49:57 +0400 Subject: [PATCH 193/405] DDP-7121 | rename instance fields based on new mapping --- .../java/org/broadinstitute/dsm/model/elastic/ESDsm.java | 8 +++----- .../model/elastic/migration/MedicalRecordMigrator.java | 2 +- .../elastic/migration/OncHistoryDetailsMigrator.java | 2 +- .../dsm/model/elastic/migration/TissueMigrator.java | 2 +- .../dsm/model/patch/OncHistoryDetailPatch.java | 4 ++-- 5 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java index 3c680d78a..23e362b3f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java @@ -1,12 +1,10 @@ package org.broadinstitute.dsm.model.elastic; import java.util.List; -import java.util.Map; import com.google.gson.annotations.SerializedName; import lombok.Getter; import org.broadinstitute.dsm.db.*; -import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantRecordDto; import org.broadinstitute.dsm.statics.ESObjectConstants; @Getter @@ -37,13 +35,13 @@ public class ESDsm { Object diagnosisYear; @SerializedName(ESObjectConstants.TISSUE_RECORDS) - List tissueRecords; // todo: change to Tissue + List tissue; // todo: change to Tissue @SerializedName(ESObjectConstants.MEDICAL_RECORDS) - List medicalRecords; + List medicalRecord; @SerializedName(ESObjectConstants.ONC_HISTORY_DETAIL_RECORDS) - List oncHistoryDetails; + List oncHistoryDetail; @SerializedName(ESObjectConstants.PARTICIPANT_DATA) List participantData; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigrator.java index 7e0e56f89..4eefefc0b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigrator.java @@ -10,7 +10,7 @@ public class MedicalRecordMigrator extends BaseCollectionMigrator { public MedicalRecordMigrator(String index, String realm) { - super(index, realm, ESObjectConstants.MEDICAL_RECORDS); + super(index, realm, "medicalRecord"); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java index abb5233b3..cf549ff81 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java @@ -11,7 +11,7 @@ public class OncHistoryDetailsMigrator extends BaseCollectionMigrator { public OncHistoryDetailsMigrator(String index, String realm) { - super(index, realm, ESObjectConstants.ONC_HISTORY_DETAILS); + super(index, realm, "oncHistoryDetail"); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigrator.java index 2a9995803..107ddc492 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigrator.java @@ -9,7 +9,7 @@ public class TissueMigrator extends BaseCollectionMigrator { public TissueMigrator(String index, String realm) { - super(index, realm, "tissues"); + super(index, realm, "tissue"); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/OncHistoryDetailPatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/OncHistoryDetailPatch.java index 63e26aad6..0632f2965 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/OncHistoryDetailPatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/OncHistoryDetailPatch.java @@ -76,7 +76,7 @@ protected Object patchNameValuePair() { Object handleSingleNameValue() { if (Patch.patch(oncHistoryDetailId, patch.getUser(), patch.getNameValue(), dbElement)) { nameValues.addAll(setWorkflowRelatedFields(patch)); - exportToESWithId(mrID.toString(), patch.getNameValue()); + exportToESWithId(oncHistoryDetailId, patch.getNameValue()); //set oncHistoryDetails created if it is a oncHistoryDetails value without a ID, otherwise created should already be set if (dbElement.getTableName().equals(DBConstants.DDP_ONC_HISTORY_DETAIL)) { NameValue oncHistoryCreated = OncHistory.setOncHistoryCreated(patch.getParentId(), patch.getUser()); @@ -92,7 +92,7 @@ Object handleSingleNameValue() { @Override Optional processEachNameValue(NameValue nameValue) { Patch.patch(oncHistoryDetailId, patch.getUser(), nameValue, dbElement); - exportToESWithId(mrID.toString(), nameValue); + exportToESWithId(oncHistoryDetailId, nameValue); return Optional.empty(); } From 903c9af15ae2399814692c91187f90832335aba5 Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 6 Dec 2021 12:09:19 +0400 Subject: [PATCH 194/405] DDP-7121 | remove Optional usage where necessary, fix minor bugs in DTO classes, refactor code based on changes --- .../dsm/db/ParticipantData.java | 20 ++++++---- .../dsm/model/at/ReceiveKitRequest.java | 2 +- .../search/ElasticSearchParticipantDto.java | 37 +++++++++++++------ .../model/participant/ParticipantWrapper.java | 4 +- .../rgp/AutomaticProbandDataCreator.java | 6 +-- .../DynamicFieldsMappingMigratorTest.java | 5 +-- 6 files changed, 45 insertions(+), 29 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/ParticipantData.java b/src/main/java/org/broadinstitute/dsm/db/ParticipantData.java index ab01187f5..29d410b66 100644 --- a/src/main/java/org/broadinstitute/dsm/db/ParticipantData.java +++ b/src/main/java/org/broadinstitute/dsm/db/ParticipantData.java @@ -1,5 +1,6 @@ package org.broadinstitute.dsm.db; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.type.TypeReference; @@ -34,6 +35,7 @@ primaryKey = DBConstants.PARTICIPANT_DATA_ID, columnPrefix = "") @JsonInclude(JsonInclude.Include.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) public class ParticipantData { private static final Logger logger = LoggerFactory.getLogger(ParticipantData.class); @@ -47,18 +49,18 @@ public class ParticipantData { "ddp_participant_id = ?, ddp_instance_id = ?, field_type_id = ?, data = ?, last_changed = ?, changed_by = ? "; @ColumnName(DBConstants.PARTICIPANT_DATA_ID) - private final String dataId; + private String participantDataId; + + @ColumnName(DBConstants.DDP_PARTICIPANT_ID) + private String ddpParticipantId; @ColumnName(DBConstants.FIELD_TYPE_ID) - private final String fieldTypeId; + private String fieldTypeId; @ColumnName (DBConstants.DATA) @JsonProperty("dynamicFields") @SerializedName("dynamicFields") - private final String data; - - @JsonProperty("dynamicFields") - public String getData() {return data;} + private String data; @JsonProperty("dynamicFields") public Map getDynamicFields() { @@ -69,8 +71,12 @@ public Map getDynamicFields() { } } + public ParticipantData() { + + } + public ParticipantData(String dataId, String fieldTypeId, String data) { - this.dataId = dataId; + this.participantDataId = dataId; this.fieldTypeId = fieldTypeId; this.data = data; } diff --git a/src/main/java/org/broadinstitute/dsm/model/at/ReceiveKitRequest.java b/src/main/java/org/broadinstitute/dsm/model/at/ReceiveKitRequest.java index 59256d069..42581da4f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/at/ReceiveKitRequest.java +++ b/src/main/java/org/broadinstitute/dsm/model/at/ReceiveKitRequest.java @@ -42,7 +42,7 @@ public static boolean receiveATKitRequest(@NonNull NotificationUtil notification data.put(GENOME_STUDY_STATUS, "3"); String dataString = new Gson().toJson(data); - if (updateData(dataString, participantData.getDataId())) { + if (updateData(dataString, participantData.getParticipantDataId())) { DDPInstance ddpInstance = DDPInstance.getDDPInstance("atcp"); List recipients = ddpInstance.getNotificationRecipient(); if (recipients != null && !recipients.isEmpty()) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchParticipantDto.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchParticipantDto.java index 15879c78d..53ec31e3b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchParticipantDto.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchParticipantDto.java @@ -1,5 +1,6 @@ package org.broadinstitute.dsm.model.elastic.search; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -19,6 +20,7 @@ public class ElasticSearchParticipantDto { private List medicalProviders; private List invitations; private List activities; + private List governedUsers; private ESComputed computed; private Long statusTimestamp; private ESProfile profile; @@ -33,16 +35,16 @@ public Optional getAddress() { return Optional.ofNullable(address); } - public Optional> getMedicalProviders() { - return Optional.ofNullable(medicalProviders); + public List getMedicalProviders() { + return medicalProviders == null ? Collections.emptyList() : medicalProviders; } - public Optional> getInvitations() { - return Optional.ofNullable(invitations); + public List getInvitations() { + return invitations == null ? Collections.emptyList() : invitations; } - public Optional> getActivities() { - return Optional.ofNullable(activities); + public List getActivities() { + return activities == null ? Collections.emptyList() : activities; } public Optional getStatusTimestamp() { @@ -53,16 +55,16 @@ public Optional getProfile() { return Optional.ofNullable(profile); } - public Optional> getFiles() { - return Optional.ofNullable(files); + public List getFiles() { + return files == null ? Collections.emptyList() : files; } - public Optional> getProxies() { - return Optional.ofNullable(proxies); + public List getProxies() { + return proxies == null ? Collections.emptyList() : proxies; } - public Optional>> getWorkflows() { - return Optional.ofNullable(workflows); + public List> getWorkflows() { + return workflows == null ? Collections.emptyList() : workflows; } public Optional getStatus() { @@ -77,6 +79,10 @@ public Optional getComputed() { return Optional.ofNullable(computed); } + public List getGovernedUsers() { + return governedUsers == null ? Collections.emptyList() : governedUsers; + } + public String getParticipantId() { return getProfile().map(esProfile -> StringUtils.isNotBlank(esProfile.getGuid()) ? esProfile.getGuid() @@ -99,6 +105,7 @@ private ElasticSearchParticipantDto(ElasticSearchParticipantDto.Builder builder) this.status = builder.status; this.dsm = builder.dsm; this.computed = builder.computed; + this.governedUsers = builder.governedUsers; } public static class Builder { @@ -107,6 +114,7 @@ public static class Builder { private List invitations; private ESComputed computed; private List activities; + private List governedUsers; private Long statusTimeStamp; private ESProfile profile; private List files; @@ -177,6 +185,11 @@ public Builder withComputed(ESComputed computed) { return this; } + public Builder withGovernedUsers(List governedUsers) { + this.governedUsers = governedUsers; + return this; + } + public ElasticSearchParticipantDto build() { return new ElasticSearchParticipantDto(this); } diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java index 9623d6fa8..d49ecf32b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java @@ -228,9 +228,9 @@ List getParticipantIdsFromElasticList(List Map> getProxiesIdsFromElasticList(List elasticSearchParticipantDtos) { Map> participantsWithProxies = new HashMap<>(); elasticSearchParticipantDtos.stream() - .filter(esParticipantData -> esParticipantData.getProxies().orElse(Collections.emptyList()).size() > 0) + .filter(esParticipantData -> esParticipantData.getProxies().size() > 0) .forEach(esParticipantData -> - participantsWithProxies.put(esParticipantData.getParticipantId(), esParticipantData.getProxies().get())); + participantsWithProxies.put(esParticipantData.getParticipantId(), esParticipantData.getProxies())); return participantsWithProxies; } diff --git a/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java b/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java index 18c55285f..43893617f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java +++ b/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java @@ -97,10 +97,8 @@ private boolean extractAndInsertProbandFromESData(DDPInstance instance, ElasticS private Map extractProbandDefaultDataFromParticipantProfile(@NonNull ElasticSearchParticipantDto esData, Optional maybeBookmark) { - Optional> participantActivities = esData.getActivities(); - String mobilePhone = participantActivities - .map(this::getPhoneNumberFromActivities) - .orElse(""); + List participantActivities = esData.getActivities(); + String mobilePhone = getPhoneNumberFromActivities(participantActivities); return esData.getProfile() .map(esProfile -> { logger.info("Starting extracting data from participant: " + esProfile.getGuid() + " ES profile"); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java index 19eaa1cac..7a2929c5b 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java @@ -1,6 +1,5 @@ package org.broadinstitute.dsm.model.elastic.migration; -import junit.framework.TestCase; import org.broadinstitute.dsm.TestHelper; import org.broadinstitute.dsm.model.elastic.export.Exportable; import org.junit.BeforeClass; @@ -18,8 +17,8 @@ public static void setUp() { @Test public void testExport() { - final String index = "participants_structured.cmi.angio"; - final String study = "angio"; + final String index = "participants_structured.atcp.atcp"; + final String study = "atcp"; List exportables = Arrays.asList( //DynamicFieldsMappingMigrator should be first in the list to make sure that mapping will be exported for first new DynamicFieldsMappingMigrator(index, study), From 22e5db7840641b734e9e6e37ef95cf1f1767c367 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 6 Dec 2021 16:18:57 +0400 Subject: [PATCH 195/405] DDP-7121 | add check for ACTIVITY display type fields --- .../DynamicFieldsMappingGenerator.java | 31 ------------ .../elastic/export/parse/TypeParser.java | 2 +- .../DynamicFieldsMappingMigrator.java | 20 ++------ .../migration/DynamicFieldsTypeParser.java | 50 +++++++++++++++++++ .../DynamicFieldsTypeParserTest.java | 29 +++++++++++ 5 files changed, 85 insertions(+), 47 deletions(-) delete mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/DynamicFieldsMappingGenerator.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParser.java create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParserTest.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/DynamicFieldsMappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/DynamicFieldsMappingGenerator.java deleted file mode 100644 index 25ec83b05..000000000 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/DynamicFieldsMappingGenerator.java +++ /dev/null @@ -1,31 +0,0 @@ -package org.broadinstitute.dsm.model.elastic.export.generate; - -import org.broadinstitute.dsm.model.elastic.export.parse.Parser; - -import java.util.HashMap; -import java.util.Map; - -public class DynamicFieldsMappingGenerator implements Generator { - - - private Parser parser; - private GeneratorPayload generatorPayload; - - - @Override - public Map generate() { - Object type = parser.parse(String.valueOf(generatorPayload.getValue())); - Map fieldWithType = new HashMap<>(Map.of(generatorPayload.getFieldName(), type)); - return new HashMap<>(Map.of("dynamicFields", Map.of("properties", fieldWithType))); - } - - @Override - public void setParser(Parser parser) { - this.parser = parser; - } - - @Override - public void setPayload(GeneratorPayload generatorPayload) { - this.generatorPayload = generatorPayload; - } -} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java index 6a6472127..45f601a0b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java @@ -22,7 +22,7 @@ public class TypeParser extends BaseParser { ))))); private static final String BOOLEAN = "boolean"; public static final Map BOOLEAN_MAPPING = new HashMap<>(Map.of(MappingGenerator.TYPE, BOOLEAN)); - private static final String DATE = "date"; + public static final String DATE = "date"; public static final Map DATE_MAPPING = new HashMap<>(Map.of(MappingGenerator.TYPE, DATE)); @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java index 3a03ad287..9f94af199 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java @@ -3,10 +3,15 @@ import static org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator.DSM_OBJECT; import static org.broadinstitute.dsm.util.ElasticSearchUtil.PROPERTIES; +import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; import org.broadinstitute.dsm.db.ParticipantData; import org.broadinstitute.dsm.db.dao.settings.FieldSettingsDao; import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; @@ -96,18 +101,3 @@ public void setRequestPayload(RequestPayload requestPayload) { } -class DynamicFieldsTypeParser extends TypeParser { - - @Override - public Object parse(String value) { - Object parsedValue; - if ("DATE".equals(value)) { - parsedValue = forDate(value); - } else if ("CHECKBOX".equals(value)) { - parsedValue = forBoolean(value); - } else { - parsedValue = forString(value); - } - return parsedValue; - } -} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParser.java new file mode 100644 index 000000000..78e6344af --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParser.java @@ -0,0 +1,50 @@ +package org.broadinstitute.dsm.model.elastic.migration; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; +import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; + +class DynamicFieldsTypeParser extends TypeParser { + + private FieldSettingsDto fieldSettingsDto; + + public void setFieldSettingsDto(FieldSettingsDto fieldSettingsDto) { + this.fieldSettingsDto = fieldSettingsDto; + } + + @Override + public Object parse(String value) { + Object parsedValue; + if ("DATE".equals(value)) { + parsedValue = forDate(value); + } else if ("CHECKBOX".equals(value)) { + parsedValue = forBoolean(value); + } else if ("ACTIVITY_STAFF".equals(value) || "ACTIVITY".equals(value)) { + String possibleValuesJson = Objects.requireNonNull(fieldSettingsDto).getPossibleValues(); + try { + List> possibleValues = new ObjectMapper().readValue(possibleValuesJson, + new TypeReference>>() { + }); + Optional maybeType = possibleValues.stream() + .filter(possibleValue -> possibleValue.containsKey("type")) + .map(possibleValue -> possibleValue.get("type")) + .findFirst(); + parsedValue = maybeType + .map(this::parse) + .orElse(forString(value)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } else { + parsedValue = forString(value); + } + return parsedValue; + } +} diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParserTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParserTest.java new file mode 100644 index 000000000..0e0d8d8bf --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParserTest.java @@ -0,0 +1,29 @@ +package org.broadinstitute.dsm.model.elastic.migration; + +import static org.junit.Assert.*; + +import java.util.Map; + +import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; +import org.broadinstitute.dsm.model.elastic.export.generate.MappingGenerator; +import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; +import org.junit.Test; + +public class DynamicFieldsTypeParserTest { + + @Test + public void parse() { + String possibleValuesJson = "[{\"value\":\"CONSENT.completedAt\",\"type\":\"DATE\"}]"; + String displayType = "ACTIVITY_STAFF"; + FieldSettingsDto fieldSettingsDto = new FieldSettingsDto.Builder(0) + .withDisplayType(displayType) + .withPossibleValues(possibleValuesJson) + .build(); + DynamicFieldsTypeParser dynamicFieldsTypeParser = new DynamicFieldsTypeParser(); + dynamicFieldsTypeParser.setFieldSettingsDto(fieldSettingsDto); + Map mapping = (Map)dynamicFieldsTypeParser.parse(displayType); + Object date = mapping.get(MappingGenerator.TYPE); + assertEquals(TypeParser.DATE, date); + } + +} \ No newline at end of file From c14dc56ab3d79734dde8c3db8335e002eb641782 Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 6 Dec 2021 17:22:30 +0400 Subject: [PATCH 196/405] DDP-7121 | change from Parser to DynamicFieldsTypeParser in DynamicFieldsMappingMigrator --- .../model/elastic/migration/DynamicFieldsMappingMigrator.java | 3 ++- .../elastic/migration/DynamicFieldsMappingMigratorTest.java | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java index 9f94af199..b1d0ea0b7 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java @@ -31,7 +31,7 @@ public class DynamicFieldsMappingMigrator implements Exportable { public static final String DYNAMIC_FIELDS_WRAPPER_NAME = "dynamicFields"; private final String index; private final String study; - public Parser parser; + public DynamicFieldsTypeParser parser; public Map propertyMap; private ElasticMappingExportAdapter elasticMappingExportAdapter; @@ -49,6 +49,7 @@ public void export() { FieldSettingsDao fieldSettingsDao = FieldSettingsDao.of(); List fieldSettingsByStudyName = fieldSettingsDao.getAllFieldSettings(); for (FieldSettingsDto fieldSettingsDto : fieldSettingsByStudyName) { + parser.setFieldSettingsDto(fieldSettingsDto); String fieldType = fieldSettingsDto.getFieldType(); BaseGenerator.PropertyInfo propertyInfo = Util.TABLE_ALIAS_MAPPINGS.get(fieldType); if (propertyInfo != null) diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java index 7a2929c5b..30aab1fe9 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java @@ -17,8 +17,8 @@ public static void setUp() { @Test public void testExport() { - final String index = "participants_structured.atcp.atcp"; - final String study = "atcp"; + final String index = "participants_structured.rgp.rgp"; + final String study = "rgp"; List exportables = Arrays.asList( //DynamicFieldsMappingMigrator should be first in the list to make sure that mapping will be exported for first new DynamicFieldsMappingMigrator(index, study), From 1b44a5270af95d7bb92f4c717ecddfa13675bf0b Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 6 Dec 2021 17:52:30 +0400 Subject: [PATCH 197/405] DDP-7121 | add special case for isApplicant --- .../org/broadinstitute/dsm/model/elastic/Util.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 16fda02ff..d90b569a1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -22,6 +22,7 @@ import org.broadinstitute.dsm.db.structure.ColumnName; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; +import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ParticipantUtil; import org.broadinstitute.dsm.util.PatchUtil; @@ -94,8 +95,8 @@ public static List> transformObjectCollectionToCollectionMap public static Map transformObjectToMap(Object obj) { Map map = new HashMap<>(); - List declaredFields = new ArrayList(List.of(obj.getClass().getDeclaredFields())); - List declaredFieldsSuper = new ArrayList(List.of(obj.getClass().getSuperclass().getDeclaredFields())); + List declaredFields = new ArrayList<>(List.of(obj.getClass().getDeclaredFields())); + List declaredFieldsSuper = new ArrayList<>(List.of(obj.getClass().getSuperclass().getDeclaredFields())); declaredFields.addAll(declaredFieldsSuper); for (Field declaredField : declaredFields) { ColumnName annotation = declaredField.getAnnotation(ColumnName.class); @@ -127,9 +128,10 @@ static Map convertToMap(String fieldName, Object fieldValue) { case "data": Map objectMap = dynamicFieldsSpecialCase(fieldValue); Map transformedMap = new HashMap<>(); - for (Map.Entry object: objectMap.entrySet()) { - String field = (String) object.getKey(); + for (Map.Entry object: objectMap.entrySet()) { + String field = object.getKey(); Object value = object.getValue(); + if (FamilyMemberConstants.IS_APPLICANT.equals(field)) value = Boolean.valueOf(String.valueOf(value)); String camelCaseField = underscoresToCamelCase(field); transformedMap.put(camelCaseField, value); } From c808580cbba4b506d2b6b76d21f31f44c3b5e6d7 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 7 Dec 2021 10:41:46 +0400 Subject: [PATCH 198/405] DDP-7121 | add singleton for object mapper --- .../elastic/export/parse/TypeParser.java | 2 +- .../migration/DynamicFieldsTypeParser.java | 31 ++++++++++++------- .../dsm/util/ObjectMapperSingleton.java | 18 +++++++++++ 3 files changed, 39 insertions(+), 12 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/util/ObjectMapperSingleton.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java index 45f601a0b..0eee33508 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java @@ -14,7 +14,7 @@ public class TypeParser extends BaseParser { private static final String TEXT = "text"; private static final String FIELDS = "fields"; private static final String KEYWORD = "keyword"; - private static final String TYPE = "type"; + protected static final String TYPE = "type"; public static final Map TEXT_KEYWORD_MAPPING = new HashMap<>( new HashMap<>( Map.of(TYPE, TEXT, diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParser.java index 78e6344af..c1ac93bad 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParser.java @@ -10,9 +10,14 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; +import org.broadinstitute.dsm.util.ObjectMapperSingleton; class DynamicFieldsTypeParser extends TypeParser { + public static final String DATE_TYPE = "DATE"; + public static final String CHECKBOX_TYPE = "CHECKBOX"; + public static final String ACTIVITY_STAFF_TYPE = "ACTIVITY_STAFF"; + public static final String ACTIVITY_TYPE = "ACTIVITY"; private FieldSettingsDto fieldSettingsDto; public void setFieldSettingsDto(FieldSettingsDto fieldSettingsDto) { @@ -20,31 +25,35 @@ public void setFieldSettingsDto(FieldSettingsDto fieldSettingsDto) { } @Override - public Object parse(String value) { + public Object parse(String type) { Object parsedValue; - if ("DATE".equals(value)) { - parsedValue = forDate(value); - } else if ("CHECKBOX".equals(value)) { - parsedValue = forBoolean(value); - } else if ("ACTIVITY_STAFF".equals(value) || "ACTIVITY".equals(value)) { + if (DATE_TYPE.equals(type)) { + parsedValue = forDate(type); + } else if (CHECKBOX_TYPE.equals(type)) { + parsedValue = forBoolean(type); + } else if (isActivityRelatedType(type)) { String possibleValuesJson = Objects.requireNonNull(fieldSettingsDto).getPossibleValues(); try { - List> possibleValues = new ObjectMapper().readValue(possibleValuesJson, + List> possibleValues = ObjectMapperSingleton.instance().readValue(possibleValuesJson, new TypeReference>>() { }); Optional maybeType = possibleValues.stream() - .filter(possibleValue -> possibleValue.containsKey("type")) - .map(possibleValue -> possibleValue.get("type")) + .filter(possibleValue -> possibleValue.containsKey(TYPE)) + .map(possibleValue -> possibleValue.get(TYPE)) .findFirst(); parsedValue = maybeType .map(this::parse) - .orElse(forString(value)); + .orElse(forString(type)); } catch (IOException e) { throw new RuntimeException(e); } } else { - parsedValue = forString(value); + parsedValue = forString(type); } return parsedValue; } + + private boolean isActivityRelatedType(String type) { + return ACTIVITY_STAFF_TYPE.equals(type) || ACTIVITY_TYPE.equals(type); + } } diff --git a/src/main/java/org/broadinstitute/dsm/util/ObjectMapperSingleton.java b/src/main/java/org/broadinstitute/dsm/util/ObjectMapperSingleton.java new file mode 100644 index 000000000..3732f70cb --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/util/ObjectMapperSingleton.java @@ -0,0 +1,18 @@ +package org.broadinstitute.dsm.util; + +import com.fasterxml.jackson.databind.ObjectMapper; + +public class ObjectMapperSingleton { + + private ObjectMapperSingleton() {} + + public static ObjectMapper instance() { + return Helper.objectMapperInstance; + } + + private static class Helper { + private static final ObjectMapper objectMapperInstance = new ObjectMapper(); + } + + +} From cb5b843dba801e8a8b8c20f7cb55a1153045351c Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 7 Dec 2021 11:07:43 +0400 Subject: [PATCH 199/405] DDP-7121 | extract two helper interfaces, extract constants, remove unused imports, refine classes, make code cleaner --- .../model/elastic/export/BaseExporter.java | 2 +- .../model/elastic/export/ExportFacade.java | 2 +- .../dsm/model/elastic/export/Exportable.java | 4 -- .../elastic/export/ExportableFactory.java | 2 +- .../elastic/export/ExportableHelper.java | 8 ++++ .../export/MappingExporterFactory.java | 5 ++- .../export/generate/BaseGenerator.java | 9 +--- .../generate/CollectionMappingGenerator.java | 4 -- .../elastic/export/generate/Generator.java | 2 - .../export/generate/GeneratorFactory.java | 2 - .../export/generate/GeneratorHelper.java | 8 ++++ .../export/generate/MappingGenerator.java | 28 ------------- .../model/elastic/export/generate/Merger.java | 9 ---- .../export/generate/NullObjectGenerator.java | 23 ---------- .../generate/SingleMappingGenerator.java | 4 -- .../generate/SourceGeneratorFactory.java | 10 ++--- .../elastic/export/parse/BaseParser.java | 10 ----- .../elastic/export/parse/TypeParser.java | 4 -- .../MedicalRecordCollectionProcessor.java | 9 ++-- .../model/elastic/migration/BaseMigrator.java | 8 +--- .../elastic/migration/BaseSingleMigrator.java | 6 +-- .../elastic/migration/BulkExportFacade.java | 12 +++--- .../DynamicFieldsMappingMigrator.java | 34 ++++----------- .../migration/DynamicFieldsTypeParser.java | 42 ++++++++++--------- .../migration/KitRequestShippingMigrator.java | 17 ++------ .../migration/MedicalRecordMigrator.java | 13 +----- .../migration/OncHistoryDetailsMigrator.java | 9 ---- .../elastic/migration/OncHistoryMigrator.java | 10 ----- .../migration/ParticipantDataMigrator.java | 11 ----- .../migration/ParticipantMigrator.java | 10 ----- .../elastic/migration/TissueMigrator.java | 18 ++------ .../dsm/statics/ESObjectConstants.java | 3 ++ .../export/generate/MappingGeneratorTest.java | 30 ------------- .../migration/BaseCollectionMigratorTest.java | 10 ----- 34 files changed, 79 insertions(+), 299 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportableHelper.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorHelper.java delete mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Merger.java delete mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/NullObjectGenerator.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java index 912e45253..561f75eea 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java @@ -6,7 +6,7 @@ import java.util.Map; import java.util.Objects; -public abstract class BaseExporter implements Exportable { +public abstract class BaseExporter implements Exportable, ExportableHelper { protected RestHighLevelClient clientInstance = ElasticSearchUtil.getClientInstance(); protected RequestPayload requestPayload; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 4d46922e1..df20afd16 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -23,7 +23,7 @@ public class ExportFacade { private static final Logger logger = LoggerFactory.getLogger(ExportFacade.class); - Exportable exportable; + BaseExporter exportable; BaseGenerator generator; ElasticSearchable searchable; private ExportFacadePayload exportFacadePayload; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java index eb075a00f..e6d615ce6 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java @@ -10,10 +10,6 @@ public interface Exportable { void export(); - void setSource(Map source); - - void setRequestPayload(RequestPayload requestPayload); - default String getParticipantGuid(String participantId, String index) { if (!(ParticipantUtil.isGuid(participantId))) { ElasticSearchParticipantDto participantById = diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportableFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportableFactory.java index 5fbdeb257..593502756 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportableFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportableFactory.java @@ -4,6 +4,6 @@ public interface ExportableFactory { - Exportable make(BaseGenerator.PropertyInfo propertyInfo); + BaseExporter make(BaseGenerator.PropertyInfo propertyInfo); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportableHelper.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportableHelper.java new file mode 100644 index 000000000..c82f1b858 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportableHelper.java @@ -0,0 +1,8 @@ +package org.broadinstitute.dsm.model.elastic.export; + +import java.util.Map; + +public interface ExportableHelper { + void setSource(Map source); + void setRequestPayload(RequestPayload requestPayload); +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingExporterFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingExporterFactory.java index 3dee6d2cf..9ca04019c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingExporterFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/MappingExporterFactory.java @@ -1,13 +1,14 @@ package org.broadinstitute.dsm.model.elastic.export; import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; +import org.broadinstitute.dsm.statics.ESObjectConstants; public class MappingExporterFactory implements ExportableFactory { @Override - public Exportable make(BaseGenerator.PropertyInfo propertyInfo) { + public BaseExporter make(BaseGenerator.PropertyInfo propertyInfo) { BaseExporter exporter = new NullObjectExporter(); - if (!propertyInfo.getFieldName().equals("followUps")) { + if (!propertyInfo.getFieldName().equals(ESObjectConstants.FOLLOW_UPS)) { exporter = new ElasticMappingExportAdapter(); } return exporter; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java index 084e6b07a..6cb16f006 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java @@ -15,7 +15,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public abstract class BaseGenerator implements Generator, Collector, Merger { +public abstract class BaseGenerator implements Generator, Collector, GeneratorHelper { private static final Logger logger = LoggerFactory.getLogger(BaseGenerator.class); @@ -57,8 +57,6 @@ protected NameValue getNameValue() { //wrap Util.getDBElement in protected method so that we can override it in testing class for tests protected DBElement getDBElement() { -// if (dbElement == null) -// dbElement = Util.getDBElement(getNameValue().getName()); return Util.getDBElement(getNameValue().getName()); } @@ -104,11 +102,6 @@ protected Object parseSingleElement() { public abstract Object construct(); - @Override - public Map merge(Map base, Map toMerge) { - return new HashMap<>(); - } - public static class PropertyInfo { private Class propertyClass; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java index 8395aff0b..8d7f4e71f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java @@ -23,8 +23,4 @@ public Map construct() { return new HashMap<>(Map.of(TYPE, NESTED, PROPERTIES, collect())); } - @Override - public Map merge(Map base, Map toMerge) { - return super.merge(base, toMerge); - } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Generator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Generator.java index 319ae34b5..4a5b64874 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Generator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Generator.java @@ -6,6 +6,4 @@ public interface Generator { Map generate(); - void setParser(Parser parser); - void setPayload(GeneratorPayload generatorPayload); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorFactory.java index 396359107..08eec6629 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorFactory.java @@ -1,7 +1,5 @@ package org.broadinstitute.dsm.model.elastic.export.generate; public interface GeneratorFactory { - BaseGenerator make(BaseGenerator.PropertyInfo propertyInfo); - } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorHelper.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorHelper.java new file mode 100644 index 000000000..c9a24f946 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorHelper.java @@ -0,0 +1,8 @@ +package org.broadinstitute.dsm.model.elastic.export.generate; + +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; + +public interface GeneratorHelper { + void setParser(Parser parser); + void setPayload(GeneratorPayload generatorPayload); +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java index 2a256aa49..a17b3166b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java @@ -37,7 +37,6 @@ public Map getCompleteMap(Object propertyMap) { return new HashMap<>(Map.of(PROPERTIES, dsmLevel)); } - @Override protected Map parseJson() { Map resultMap = new HashMap<>(); @@ -49,31 +48,4 @@ protected Map parseJson() { Map returnMap = new HashMap<>(Map.of("dynamicFields", new HashMap<>(Map.of(PROPERTIES, resultMap)))); return returnMap; } - - @Override - public Map merge(Map base, Map toMerge) { - String propertyName = Util.underscoresToCamelCase(getDBElement().getColumnName()); -// base.putIfAbsent(propertyName, toMerge); - - - - //dsm -> properties -> medicalRecords -> properties -> mrRecordId: {}, ragaca: {}, additionalValuesjson: { properties: {key: {type = ...} - getFieldLevel(base).putAll(getFieldLevel(toMerge)); - return base; - } - - private Map getFieldLevel(Map topLevel) { - if (topLevel.isEmpty()) { - return topLevel; - } - String propertyName = Util.underscoresToCamelCase(getDBElement().getColumnName()); -// Map outerPropertyMap = (Map) topLevel.get(PROPERTIES); - Map innerPropertyMap = (Map) topLevel.get(propertyName); - Map innerProperty = (Map) innerPropertyMap.get(PROPERTIES); - if (innerProperty != null) - return innerProperty; - else - return innerPropertyMap; - } - } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Merger.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Merger.java deleted file mode 100644 index 7f5c71454..000000000 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Merger.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.broadinstitute.dsm.model.elastic.export.generate; - -import java.util.Map; - -public interface Merger { - - Map merge(Map base, Map toMerge); - -} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/NullObjectGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/NullObjectGenerator.java deleted file mode 100644 index 6457a9d3f..000000000 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/NullObjectGenerator.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.broadinstitute.dsm.model.elastic.export.generate; - -import java.util.HashMap; -import java.util.Map; - -import org.broadinstitute.dsm.model.elastic.export.parse.Parser; - -public class NullObjectGenerator implements Generator { - @Override - public Map generate() { - return new HashMap<>(); - } - - @Override - public void setParser(Parser parser) { - - } - - @Override - public void setPayload(GeneratorPayload generatorPayload) { - - } -} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java index a96dc7525..963f8e390 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java @@ -1,13 +1,9 @@ package org.broadinstitute.dsm.model.elastic.export.generate; -import org.broadinstitute.dsm.model.NameValue; -import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; -import org.jruby.ir.operands.Hash; import java.util.HashMap; import java.util.Map; -import java.util.Objects; public class SingleMappingGenerator extends MappingGenerator { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorFactory.java index c5e0866af..17321048e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorFactory.java @@ -4,12 +4,8 @@ public class SourceGeneratorFactory implements GeneratorFactory { @Override public BaseGenerator make(BaseGenerator.PropertyInfo propertyInfo) { - BaseGenerator generator; - if (propertyInfo.isCollection()) { - generator = new CollectionSourceGenerator(); - } else { - generator = new SingleSourceGenerator(); - } - return generator; + return propertyInfo.isCollection() + ? new CollectionSourceGenerator() + : new SingleSourceGenerator(); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java index bb6300d12..6485c2dec 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java @@ -70,14 +70,4 @@ boolean isBoolean(String value) { return value.equalsIgnoreCase("true") || value.equalsIgnoreCase("false"); } - - boolean isCollection(String value) { - boolean isCollection = false; - try { - new Gson().fromJson(value, List.class); - isCollection = true; - } catch (JsonSyntaxException ignored) { - } - return isCollection; - } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java index 0eee33508..3d68e0aec 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java @@ -1,16 +1,12 @@ package org.broadinstitute.dsm.model.elastic.export.parse; -import com.google.common.reflect.TypeToken; -import com.google.gson.Gson; import org.broadinstitute.dsm.model.elastic.export.generate.MappingGenerator; import java.util.HashMap; -import java.util.List; import java.util.Map; public class TypeParser extends BaseParser { - private static final String TEXT = "text"; private static final String FIELDS = "fields"; private static final String KEYWORD = "keyword"; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/MedicalRecordCollectionProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/MedicalRecordCollectionProcessor.java index 6c8603001..8a4e27880 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/MedicalRecordCollectionProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/MedicalRecordCollectionProcessor.java @@ -7,12 +7,9 @@ import com.google.gson.Gson; import org.broadinstitute.dsm.model.elastic.ESDsm; import org.broadinstitute.dsm.model.elastic.export.generate.Collector; +import org.broadinstitute.dsm.statics.ESObjectConstants; public class MedicalRecordCollectionProcessor extends CollectionProcessor { - public MedicalRecordCollectionProcessor(ESDsm esDsm, String propertyName, int recordId, - Collector collector) { - super(esDsm, propertyName, recordId, collector); - } public MedicalRecordCollectionProcessor() { @@ -21,9 +18,9 @@ public MedicalRecordCollectionProcessor() { @Override protected List> updateIfExistsOrPut(List> fetchedRecords) { for (Map medicalRecord: fetchedRecords) { - Object followUps = medicalRecord.get("followUps"); + Object followUps = medicalRecord.get(ESObjectConstants.FOLLOW_UPS); if (!Objects.isNull(followUps)) { - medicalRecord.put("followUps", new Gson().toJson(followUps)); + medicalRecord.put(ESObjectConstants.FOLLOW_UPS, new Gson().toJson(followUps)); } } return super.updateIfExistsOrPut(fetchedRecords); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseMigrator.java index f6d08ed78..e575ab45c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseMigrator.java @@ -1,17 +1,13 @@ package org.broadinstitute.dsm.model.elastic.migration; -import java.util.Map; - import org.broadinstitute.dsm.model.elastic.export.BaseExporter; -import org.broadinstitute.dsm.model.elastic.export.ElasticMappingExportAdapter; -import org.broadinstitute.dsm.model.elastic.export.Exportable; -import org.broadinstitute.dsm.model.elastic.export.RequestPayload; -import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; import org.broadinstitute.dsm.model.elastic.export.generate.Generator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import spark.utils.StringUtils; +import java.util.Map; + public abstract class BaseMigrator extends BaseExporter implements Generator { private static final Logger logger = LoggerFactory.getLogger(BaseMigrator.class); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseSingleMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseSingleMigrator.java index e4428221b..629c5aa5f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseSingleMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseSingleMigrator.java @@ -1,11 +1,11 @@ package org.broadinstitute.dsm.model.elastic.migration; -import java.util.HashMap; -import java.util.Map; - import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.statics.ESObjectConstants; +import java.util.HashMap; +import java.util.Map; + public abstract class BaseSingleMigrator extends BaseMigrator { private Map transformedObject; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BulkExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BulkExportFacade.java index 1988a9c72..e2909cb5c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BulkExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BulkExportFacade.java @@ -1,20 +1,18 @@ package org.broadinstitute.dsm.model.elastic.migration; -import static org.broadinstitute.dsm.model.elastic.Util.DOC; - -import java.io.IOException; -import java.util.List; -import java.util.Map; - import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.elasticsearch.action.bulk.BulkRequest; -import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.util.Map; + +import static org.broadinstitute.dsm.model.elastic.Util.DOC; + public class BulkExportFacade { private static final Logger logger = LoggerFactory.getLogger(BulkExportFacade.class); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java index b1d0ea0b7..336116652 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java @@ -1,17 +1,5 @@ package org.broadinstitute.dsm.model.elastic.migration; -import static org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator.DSM_OBJECT; -import static org.broadinstitute.dsm.util.ElasticSearchUtil.PROPERTIES; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; import org.broadinstitute.dsm.db.ParticipantData; import org.broadinstitute.dsm.db.dao.settings.FieldSettingsDao; import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; @@ -21,10 +9,13 @@ import org.broadinstitute.dsm.model.elastic.export.RequestPayload; import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; import org.broadinstitute.dsm.model.elastic.export.generate.MappingGenerator; -import org.broadinstitute.dsm.model.elastic.export.parse.Parser; -import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; -import org.broadinstitute.dsm.statics.DBConstants; -import org.broadinstitute.dsm.statics.ESObjectConstants; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator.DSM_OBJECT; +import static org.broadinstitute.dsm.util.ElasticSearchUtil.PROPERTIES; public class DynamicFieldsMappingMigrator implements Exportable { @@ -89,16 +80,5 @@ private void buildMapping(FieldSettingsDto fieldSettingsDto, BaseGenerator.Prope innerProperties.putIfAbsent(columnName, typeMap); } } - - @Override - public void setSource(Map source) { - - } - - @Override - public void setRequestPayload(RequestPayload requestPayload) { - - } - } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParser.java index c1ac93bad..e2b398955 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParser.java @@ -1,17 +1,16 @@ package org.broadinstitute.dsm.model.elastic.migration; +import com.fasterxml.jackson.core.type.TypeReference; +import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; +import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; +import org.broadinstitute.dsm.util.ObjectMapperSingleton; + import java.io.IOException; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; -import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; -import org.broadinstitute.dsm.util.ObjectMapperSingleton; - class DynamicFieldsTypeParser extends TypeParser { public static final String DATE_TYPE = "DATE"; @@ -33,26 +32,29 @@ public Object parse(String type) { parsedValue = forBoolean(type); } else if (isActivityRelatedType(type)) { String possibleValuesJson = Objects.requireNonNull(fieldSettingsDto).getPossibleValues(); - try { - List> possibleValues = ObjectMapperSingleton.instance().readValue(possibleValuesJson, - new TypeReference>>() { - }); - Optional maybeType = possibleValues.stream() - .filter(possibleValue -> possibleValue.containsKey(TYPE)) - .map(possibleValue -> possibleValue.get(TYPE)) - .findFirst(); - parsedValue = maybeType - .map(this::parse) - .orElse(forString(type)); - } catch (IOException e) { - throw new RuntimeException(e); - } + Optional maybeType = getTypeFromPossibleValuesJson(possibleValuesJson); + parsedValue = maybeType + .map(this::parse) + .orElse(forString(type)); } else { parsedValue = forString(type); } return parsedValue; } + private Optional getTypeFromPossibleValuesJson(String possibleValuesJson) { + try { + List> possibleValues = ObjectMapperSingleton.instance().readValue(possibleValuesJson, new TypeReference>>() {}); + Optional maybeType = possibleValues.stream() + .filter(possibleValue -> possibleValue.containsKey(TYPE)) + .map(possibleValue -> possibleValue.get(TYPE)) + .findFirst(); + return maybeType; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + private boolean isActivityRelatedType(String type) { return ACTIVITY_STAFF_TYPE.equals(type) || ACTIVITY_TYPE.equals(type); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/KitRequestShippingMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/KitRequestShippingMigrator.java index 8969f527f..2fcaa610a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/KitRequestShippingMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/KitRequestShippingMigrator.java @@ -1,13 +1,11 @@ package org.broadinstitute.dsm.model.elastic.migration; -import java.util.List; -import java.util.Map; - import org.broadinstitute.dsm.db.KitRequestShipping; -import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; -import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import org.broadinstitute.dsm.statics.ESObjectConstants; +import java.util.List; +import java.util.Map; + public class KitRequestShippingMigrator extends BaseCollectionMigrator { public KitRequestShippingMigrator(String index, String realm) { @@ -20,13 +18,4 @@ protected Map getDataByRealm() { return (Map) kitRequests; } - @Override - public void setParser(Parser parser) { - - } - - @Override - public void setPayload(GeneratorPayload generatorPayload) { - - } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigrator.java index 4eefefc0b..72e4572a1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigrator.java @@ -1,8 +1,6 @@ package org.broadinstitute.dsm.model.elastic.migration; import org.broadinstitute.dsm.db.MedicalRecord; -import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; -import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import org.broadinstitute.dsm.statics.ESObjectConstants; import java.util.Map; @@ -10,7 +8,7 @@ public class MedicalRecordMigrator extends BaseCollectionMigrator { public MedicalRecordMigrator(String index, String realm) { - super(index, realm, "medicalRecord"); + super(index, realm, ESObjectConstants.MEDICAL_RECORD); } @Override @@ -18,13 +16,4 @@ protected Map getDataByRealm() { return (Map) MedicalRecord.getMedicalRecords(realm); } - @Override - public void setParser(Parser parser) { - - } - - @Override - public void setPayload(GeneratorPayload generatorPayload) { - - } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java index cf549ff81..a79b46ddc 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java @@ -19,13 +19,4 @@ protected Map getDataByRealm() { return (Map) OncHistoryDetail.getOncHistoryDetails(realm); } - @Override - public void setParser(Parser parser) { - - } - - @Override - public void setPayload(GeneratorPayload generatorPayload) { - - } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryMigrator.java index 48656c773..3f2a53e94 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryMigrator.java @@ -17,14 +17,4 @@ public OncHistoryMigrator(String index, String realm) { protected Map getDataByRealm() { return (Map) new OncHistoryDao().getOncHistoriesByStudy(realm); } - - @Override - public void setParser(Parser parser) { - - } - - @Override - public void setPayload(GeneratorPayload generatorPayload) { - - } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java index b1bace846..816bfa48b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java @@ -24,15 +24,4 @@ protected Map getDataByRealm() { Map> participantDataByRealm = participantDataDao.getParticipantDataByRealm(realm); return (Map) participantDataByRealm; } - - - @Override - public void setParser(Parser parser) { - - } - - @Override - public void setPayload(GeneratorPayload generatorPayload) { - - } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantMigrator.java index 6d718171d..19f4807e3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantMigrator.java @@ -20,14 +20,4 @@ public ParticipantMigrator(String index, String realm) { protected Map getDataByRealm() { return (Map) Participant.getParticipants(realm); } - - @Override - public void setParser(Parser parser) { - - } - - @Override - public void setPayload(GeneratorPayload generatorPayload) { - - } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigrator.java index 107ddc492..7e1ebf68e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigrator.java @@ -1,15 +1,14 @@ package org.broadinstitute.dsm.model.elastic.migration; -import java.util.Map; - import org.broadinstitute.dsm.db.dao.ddp.tissue.TissueDao; -import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; -import org.broadinstitute.dsm.model.elastic.export.parse.Parser; +import org.broadinstitute.dsm.statics.ESObjectConstants; + +import java.util.Map; public class TissueMigrator extends BaseCollectionMigrator { public TissueMigrator(String index, String realm) { - super(index, realm, "tissue"); + super(index, realm, ESObjectConstants.TISSUE); } @Override @@ -17,13 +16,4 @@ protected Map getDataByRealm() { return (Map) new TissueDao().getTissuesByStudy(realm); } - @Override - public void setParser(Parser parser) { - - } - - @Override - public void setPayload(GeneratorPayload generatorPayload) { - - } } diff --git a/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java b/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java index 0b4e80897..676f8ae36 100644 --- a/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java +++ b/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java @@ -18,12 +18,15 @@ public class ESObjectConstants { //medical records public static final String MEDICAL_RECORDS = "medicalRecords"; + public static final String MEDICAL_RECORD = "medicalRecord"; public static final String MEDICAL_RECORDS_ID = "medicalRecordId"; + public static final String FOLLOW_UPS = "followUps"; public static final List MEDICAL_RECORDS_FIELD_NAMES = Arrays.asList("name", "type", "requested", "received"); //tissue records public static final String TISSUE_RECORDS = "tissueRecords"; public static final String TISSUE_RECORDS_ID = "tissueRecordId"; + public static final String TISSUE = "tissue"; public static final List TISSUE_RECORDS_FIELD_NAMES = Arrays.asList("typePX", "locationPX", "datePX", "histology", "accessionNumber", "requested", "received", "sent"); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java index e309602da..7b310d2da 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java @@ -94,36 +94,6 @@ public void generateMapping() { Assert.assertTrue(topLevel.equals(resultMap)); } - @Test - public void merge() { - NameValue nameValue = new NameValue("d.data", "{\"DDP_INSTANCE\": \"TEST\"}"); - GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, 0); - TypeParser parser = new TypeParser(); - MappingGenerator mappingGenerator = new CollectionMappingGenerator(parser, generatorPayload); - Map base = new HashMap<>(); - base = mappingGenerator.merge(base, mappingGenerator.generate()); - NameValue nameValue2 = new NameValue("d.data", "{\"DDP_INSTANCE1\": \"TEST1\"}"); - GeneratorPayload generatorPayload2 = new GeneratorPayload(nameValue2, 0); - mappingGenerator = new CollectionMappingGenerator(parser, generatorPayload2); - base = mappingGenerator.merge(base, mappingGenerator.generate()); - Object value = ((Map) ((Map) ((Map) ((Map) ((Map) base - .get(PROPERTIES)) - .get(MappingGenerator.DSM_OBJECT)) - .get(PROPERTIES)) - .get(mappingGenerator.getPropertyName())) - .get(PROPERTIES)) - .get(Util.underscoresToCamelCase("DDP_INSTANCE1")); - Object value2 = ((Map) ((Map) ((Map) ((Map) ((Map) base - .get(PROPERTIES)) - .get(MappingGenerator.DSM_OBJECT)) - .get(PROPERTIES)) - .get(mappingGenerator.getPropertyName())) - .get(PROPERTIES)) - .get(Util.underscoresToCamelCase("DDP_INSTANCE")); - Assert.assertFalse(Objects.isNull(value)); - Assert.assertFalse(Objects.isNull(value2)); - } - @Test public void parseJson() { NameValue nameValue = new NameValue("m.additionalValuesJson", "{\"DDP_INSTANCE\": \"TEST\", \"DDP_INSTANCE1\": \"TEST1\"}"); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigratorTest.java index 196934bf6..f1c0e2ef5 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigratorTest.java @@ -51,15 +51,5 @@ public MockBaseCollectionMigrator(String index, String realm, String object) { protected Map getDataByRealm() { return Map.of(); } - - @Override - public void setParser(Parser parser) { - - } - - @Override - public void setPayload(GeneratorPayload generatorPayload) { - - } } } \ No newline at end of file From 8073956251f46a76e2af828d0f4396768135df8f Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 7 Dec 2021 11:42:44 +0400 Subject: [PATCH 200/405] DDP-7121 | refactoring hardcoded literals --- .../broadinstitute/dsm/model/elastic/Util.java | 15 ++++++++------- .../dsm/model/elastic/export/BaseExporter.java | 5 ++--- .../elastic/export/ElasticDataExportAdapter.java | 1 - .../export/ElasticMappingExportAdapter.java | 1 - .../dsm/model/elastic/export/ExportFacade.java | 9 +++++++-- .../model/elastic/export/ExportFacadePayload.java | 1 - .../dsm/model/elastic/export/Exportable.java | 2 -- .../dsm/model/elastic/export/RequestPayload.java | 4 +++- .../elastic/export/generate/BaseGenerator.java | 1 - .../generate/CollectionSourceGenerator.java | 4 ++-- .../model/elastic/export/generate/Generator.java | 2 -- .../elastic/export/generate/MappingGenerator.java | 10 +++++----- .../export/generate/SingleMappingGenerator.java | 4 ++-- .../export/generate/SingleSourceGenerator.java | 9 +-------- .../elastic/export/generate/SourceGenerator.java | 11 ++++------- .../model/elastic/export/parse/BaseParser.java | 9 ++------- .../model/elastic/export/parse/TypeParser.java | 4 ++-- .../elastic/export/process/BaseProcessor.java | 4 ++-- .../export/process/CollectionProcessor.java | 1 - .../process/MedicalRecordCollectionProcessor.java | 2 -- .../elastic/migration/BaseCollectionMigrator.java | 9 ++++++--- .../migration/OncHistoryDetailsMigrator.java | 4 +--- .../elastic/migration/OncHistoryMigrator.java | 2 -- .../migration/ParticipantDataMigrator.java | 2 -- .../elastic/migration/ParticipantMigrator.java | 3 --- .../model/elastic/migration/TissueMigrator.java | 4 ++-- .../dsm/statics/ESObjectConstants.java | 3 ++- 27 files changed, 51 insertions(+), 75 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index d90b569a1..ce2b10cc7 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -23,6 +23,7 @@ import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; +import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ParticipantUtil; import org.broadinstitute.dsm.util.PatchUtil; @@ -30,13 +31,13 @@ public class Util { public static final Map TABLE_ALIAS_MAPPINGS = Map.of( - "m", new BaseGenerator.PropertyInfo(MedicalRecord.class, true), - "t", new BaseGenerator.PropertyInfo(Tissue.class, true), - "oD", new BaseGenerator.PropertyInfo(OncHistoryDetail.class, true), - "d", new BaseGenerator.PropertyInfo(ParticipantData.class, true), - "r", new BaseGenerator.PropertyInfo(Participant.class, false), - "p", new BaseGenerator.PropertyInfo(Participant.class, false), - "o", new BaseGenerator.PropertyInfo(OncHistory.class, false) + DBConstants.DDP_MEDICAL_RECORD_ALIAS, new BaseGenerator.PropertyInfo(MedicalRecord.class, true), + DBConstants.DDP_TISSUE_ALIAS, new BaseGenerator.PropertyInfo(Tissue.class, true), + DBConstants.DDP_ONC_HISTORY_DETAIL_ALIAS, new BaseGenerator.PropertyInfo(OncHistoryDetail.class, true), + DBConstants.DDP_PARTICIPANT_DATA_ALIAS, new BaseGenerator.PropertyInfo(ParticipantData.class, true), + DBConstants.DDP_PARTICIPANT_RECORD_ALIAS, new BaseGenerator.PropertyInfo(Participant.class, false), + DBConstants.DDP_PARTICIPANT_ALIAS, new BaseGenerator.PropertyInfo(Participant.class, false), + DBConstants.DDP_ONC_HISTORY_ALIAS, new BaseGenerator.PropertyInfo(OncHistory.class, false) ); public static final int FIRST_ELEMENT_INDEX = 0; public static final String UNDERSCORE_SEPARATOR = "_"; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java index 561f75eea..8d19c62eb 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java @@ -1,11 +1,10 @@ package org.broadinstitute.dsm.model.elastic.export; +import java.util.Map; + import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.elasticsearch.client.RestHighLevelClient; -import java.util.Map; -import java.util.Objects; - public abstract class BaseExporter implements Exportable, ExportableHelper { protected RestHighLevelClient clientInstance = ElasticSearchUtil.getClientInstance(); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticDataExportAdapter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticDataExportAdapter.java index a8364de58..c0a27d49f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticDataExportAdapter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticDataExportAdapter.java @@ -1,7 +1,6 @@ package org.broadinstitute.dsm.model.elastic.export; import java.io.IOException; -import java.util.Map; import org.broadinstitute.dsm.model.elastic.Util; import org.elasticsearch.action.update.UpdateRequest; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticMappingExportAdapter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticMappingExportAdapter.java index a7712d94b..88bab66b0 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticMappingExportAdapter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticMappingExportAdapter.java @@ -1,7 +1,6 @@ package org.broadinstitute.dsm.model.elastic.export; import java.io.IOException; -import java.util.Map; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.indices.PutMappingRequest; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index df20afd16..135acf3c2 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -8,10 +8,15 @@ import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.elastic.ESDsm; import org.broadinstitute.dsm.model.elastic.Util; -import org.broadinstitute.dsm.model.elastic.export.generate.*; +import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; +import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorFactory; +import org.broadinstitute.dsm.model.elastic.export.generate.MappingGenerator; +import org.broadinstitute.dsm.model.elastic.export.generate.MappingGeneratorFactory; +import org.broadinstitute.dsm.model.elastic.export.generate.SourceGeneratorFactory; import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; import org.broadinstitute.dsm.model.elastic.export.parse.ValueParser; -import org.broadinstitute.dsm.model.elastic.export.process.*; +import org.broadinstitute.dsm.model.elastic.export.process.BaseProcessor; +import org.broadinstitute.dsm.model.elastic.export.process.CollectionProcessorFactory; import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchable; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java index 58f436644..27741a59c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java @@ -3,7 +3,6 @@ import java.util.Objects; import org.broadinstitute.dsm.db.structure.DBElement; -import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; import org.broadinstitute.dsm.util.PatchUtil; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java index e6d615ce6..efaeb8d0a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java @@ -1,7 +1,5 @@ package org.broadinstitute.dsm.model.elastic.export; -import java.util.Map; - import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.util.ParticipantUtil; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/RequestPayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/RequestPayload.java index 976310686..d63620fd4 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/RequestPayload.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/RequestPayload.java @@ -1,5 +1,7 @@ package org.broadinstitute.dsm.model.elastic.export; +import org.apache.commons.lang3.StringUtils; + public class RequestPayload { private String index; @@ -11,7 +13,7 @@ public RequestPayload(String index, String id) { } public RequestPayload(String index) { - this(index, ""); + this(index, StringUtils.EMPTY); } public String getIndex() { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java index 6cb16f006..4f33b8371 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java @@ -1,6 +1,5 @@ package org.broadinstitute.dsm.model.elastic.export.generate; -import java.util.HashMap; import java.util.Map; import java.util.Objects; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java index 2bac83c66..9bb63c081 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java @@ -4,8 +4,8 @@ import java.util.List; import java.util.Map; -import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; +import org.broadinstitute.dsm.statics.ESObjectConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,7 +27,7 @@ public Object construct() { logger.info("Constructing nested data"); Map mapWithParsedObjects = new HashMap<>(Map.of( getPrimaryKey(), generatorPayload.getRecordId(), - "dynamicFields", parseJsonValuesToObject()) + ESObjectConstants.DYNAMIC_FIELDS, parseJsonValuesToObject()) ); return List.of(mapWithParsedObjects); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Generator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Generator.java index 4a5b64874..01cc63886 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Generator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Generator.java @@ -2,8 +2,6 @@ import java.util.Map; -import org.broadinstitute.dsm.model.elastic.export.parse.Parser; - public interface Generator { Map generate(); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java index a17b3166b..6e690f8ee 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java @@ -1,14 +1,14 @@ package org.broadinstitute.dsm.model.elastic.export.generate; +import java.util.HashMap; +import java.util.Map; + import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; +import org.broadinstitute.dsm.statics.ESObjectConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; - abstract public class MappingGenerator extends BaseGenerator { private static final Logger logger = LoggerFactory.getLogger(MappingGenerator.class); @@ -45,7 +45,7 @@ protected Map parseJson() { Object eachType = parser.parse(String.valueOf(entry.getValue())); resultMap.put(Util.underscoresToCamelCase(entry.getKey()), eachType); } - Map returnMap = new HashMap<>(Map.of("dynamicFields", new HashMap<>(Map.of(PROPERTIES, resultMap)))); + Map returnMap = new HashMap<>(Map.of(ESObjectConstants.DYNAMIC_FIELDS, new HashMap<>(Map.of(PROPERTIES, resultMap)))); return returnMap; } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java index 963f8e390..79b58029b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleMappingGenerator.java @@ -1,10 +1,10 @@ package org.broadinstitute.dsm.model.elastic.export.generate; -import org.broadinstitute.dsm.model.elastic.export.parse.Parser; - import java.util.HashMap; import java.util.Map; +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; + public class SingleMappingGenerator extends MappingGenerator { public SingleMappingGenerator(Parser parser, GeneratorPayload generatorPayload) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleSourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleSourceGenerator.java index 4e5740a88..2b1a974b1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleSourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleSourceGenerator.java @@ -12,14 +12,7 @@ public class SingleSourceGenerator extends SourceGenerator { private static final Logger logger = LoggerFactory.getLogger(SingleSourceGenerator.class); - public SingleSourceGenerator(Parser parser, - GeneratorPayload generatorPayload) { - super(parser, generatorPayload); - } - - public SingleSourceGenerator() { - - } + public SingleSourceGenerator() {} @Override public Object construct() { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java index 9e285d69d..b409f040c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java @@ -1,14 +1,13 @@ package org.broadinstitute.dsm.model.elastic.export.generate; +import java.util.HashMap; +import java.util.Map; + import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - abstract public class SourceGenerator extends BaseGenerator { private static final Logger logger = LoggerFactory.getLogger(SourceGenerator.class); @@ -17,9 +16,7 @@ public SourceGenerator(Parser parser, GeneratorPayload generatorPayload) { super(parser, generatorPayload); } - public SourceGenerator() { - - } + public SourceGenerator() {} @Override public Map generate() { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java index 6485c2dec..0ce01fd82 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java @@ -4,12 +4,7 @@ import java.time.LocalDateTime; import java.time.LocalTime; import java.time.format.DateTimeParseException; -import java.util.List; -import java.util.Map; -import com.google.common.reflect.TypeToken; -import com.google.gson.Gson; -import com.google.gson.JsonSyntaxException; import org.apache.commons.lang3.StringUtils; public abstract class BaseParser implements Parser { @@ -67,7 +62,7 @@ boolean isDate(String value) { } boolean isBoolean(String value) { - return value.equalsIgnoreCase("true") || - value.equalsIgnoreCase("false"); + return value.equalsIgnoreCase(Boolean.TRUE.toString()) || + value.equalsIgnoreCase(Boolean.FALSE.toString()); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java index 3d68e0aec..b49093ec1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java @@ -1,10 +1,10 @@ package org.broadinstitute.dsm.model.elastic.export.parse; -import org.broadinstitute.dsm.model.elastic.export.generate.MappingGenerator; - import java.util.HashMap; import java.util.Map; +import org.broadinstitute.dsm.model.elastic.export.generate.MappingGenerator; + public class TypeParser extends BaseParser { private static final String TEXT = "text"; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/BaseProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/BaseProcessor.java index 33a48c410..bc58116d3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/BaseProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/BaseProcessor.java @@ -1,10 +1,10 @@ package org.broadinstitute.dsm.model.elastic.export.process; +import java.util.Objects; + import org.broadinstitute.dsm.model.elastic.ESDsm; import org.broadinstitute.dsm.model.elastic.export.generate.Collector; -import java.util.Objects; - public abstract class BaseProcessor implements Processor { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java index 8d5f094ef..ae989ba98 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java @@ -13,7 +13,6 @@ import org.broadinstitute.dsm.model.elastic.ESDsm; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.generate.Collector; -import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/MedicalRecordCollectionProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/MedicalRecordCollectionProcessor.java index 8a4e27880..d0b754557 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/MedicalRecordCollectionProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/MedicalRecordCollectionProcessor.java @@ -5,8 +5,6 @@ import java.util.Objects; import com.google.gson.Gson; -import org.broadinstitute.dsm.model.elastic.ESDsm; -import org.broadinstitute.dsm.model.elastic.export.generate.Collector; import org.broadinstitute.dsm.statics.ESObjectConstants; public class MedicalRecordCollectionProcessor extends CollectionProcessor { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java index 6af5e7184..2c577c5ae 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java @@ -1,11 +1,14 @@ package org.broadinstitute.dsm.model.elastic.migration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.statics.ESObjectConstants; -import java.util.*; -import java.util.stream.Collectors; - public abstract class BaseCollectionMigrator extends BaseMigrator { protected List> transformedList; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java index a79b46ddc..6c14b5534 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigrator.java @@ -3,15 +3,13 @@ import java.util.Map; import org.broadinstitute.dsm.db.OncHistoryDetail; -import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; -import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import org.broadinstitute.dsm.statics.ESObjectConstants; public class OncHistoryDetailsMigrator extends BaseCollectionMigrator { public OncHistoryDetailsMigrator(String index, String realm) { - super(index, realm, "oncHistoryDetail"); + super(index, realm, ESObjectConstants.ONC_HISTORY_DETAIL); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryMigrator.java index 3f2a53e94..943bc91af 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryMigrator.java @@ -3,8 +3,6 @@ import java.util.Map; import org.broadinstitute.dsm.db.dao.ddp.onchistory.OncHistoryDao; -import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; -import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import org.broadinstitute.dsm.statics.ESObjectConstants; public class OncHistoryMigrator extends BaseSingleMigrator { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java index 816bfa48b..660163231 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java @@ -6,8 +6,6 @@ import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; -import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; -import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import org.broadinstitute.dsm.statics.ESObjectConstants; public class ParticipantDataMigrator extends BaseCollectionMigrator { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantMigrator.java index 19f4807e3..67871a1fb 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantMigrator.java @@ -3,11 +3,8 @@ import java.util.Map; import org.broadinstitute.dsm.db.Participant; -import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.Exportable; import org.broadinstitute.dsm.model.elastic.export.generate.Generator; -import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; -import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import org.broadinstitute.dsm.statics.ESObjectConstants; public class ParticipantMigrator extends BaseSingleMigrator implements Exportable, Generator { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigrator.java index 7e1ebf68e..fef64a756 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigrator.java @@ -1,10 +1,10 @@ package org.broadinstitute.dsm.model.elastic.migration; +import java.util.Map; + import org.broadinstitute.dsm.db.dao.ddp.tissue.TissueDao; import org.broadinstitute.dsm.statics.ESObjectConstants; -import java.util.Map; - public class TissueMigrator extends BaseCollectionMigrator { public TissueMigrator(String index, String realm) { diff --git a/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java b/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java index 676f8ae36..b584f6bab 100644 --- a/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java +++ b/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java @@ -70,9 +70,10 @@ public class ESObjectConstants { public static final String HAS_CONSENTED_TO_TISSUE_SAMPLE = "hasConsentedToTissueSample"; public static final String DIAGNOSIS_YEAR = "diagnosisYear"; public static final String ONC_HISTORY_DETAIL_RECORDS = "oncHistoryDetails"; - public static final String ONC_HISTORY_DETAILS = "oncHistoryDetails"; + public static final String ONC_HISTORY_DETAIL = "oncHistoryDetail"; public static final String ONC_HISTORY = "oncHistory"; public static final String PARTICIPANT_DATA = "participantData"; public static final String PARTICIPANT_RECORD = "participantRecord"; public static final String PARTICIPANT = "participant"; + public static final String DYNAMIC_FIELDS = "dynamicFields"; } From 7809a6d4fd3862ea916b61b7e7810463dd310ab9 Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 7 Dec 2021 16:57:42 +0400 Subject: [PATCH 201/405] DDP-7121 | start reducing the size of serialize() method in ElasticSearch.java by refactoring it --- .../model/elastic/search/ElasticSearch.java | 125 +++++++++--------- 1 file changed, 65 insertions(+), 60 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java index 624f0ee5a..6fc8c72c9 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java @@ -1,7 +1,6 @@ package org.broadinstitute.dsm.model.elastic.search; import java.io.IOException; -import java.io.StringReader; import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; @@ -11,22 +10,19 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; -import com.google.gson.stream.JsonReader; import lombok.Setter; import org.apache.commons.lang3.StringUtils; -import org.broadinstitute.dsm.db.MedicalRecord; -import org.broadinstitute.dsm.model.FollowUp; import org.broadinstitute.dsm.model.elastic.ESDsm; import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; +import org.broadinstitute.dsm.util.ObjectMapperSingleton; import org.broadinstitute.dsm.util.ParticipantUtil; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; @@ -74,75 +70,84 @@ public long getTotalCount() { public static Optional parseSourceMap(Map sourceMap) { if (sourceMap == null) return Optional.of(new ElasticSearchParticipantDto.Builder().build()); - ElasticSearchParticipantDto elasticSearchParticipantDto = serialize(sourceMap); + ElasticSearchParticipantDto elasticSearchParticipantDto = deserialize(sourceMap); return Optional.of(elasticSearchParticipantDto); } - private static ElasticSearchParticipantDto serialize(Map sourceMap) { + private static ElasticSearchParticipantDto deserialize(Map sourceMap) { ElasticSearchParticipantDto elasticSearchParticipantDto = null; - Object dsm = sourceMap.get(DSM); - ObjectMapper objectMapper = new ObjectMapper(); - if (!Objects.isNull(dsm)) { - Map dsmLevel = (Map) sourceMap.get(DSM); - Map clonedProperty = new HashMap<>(); - for (Map.Entry entry: dsmLevel.entrySet()) { - String key = entry.getKey(); //medicalRecords - Object value = entry.getValue(); //medicalRecords: [] - Field property = null; - try { - property = ESDsm.class.getDeclaredField(key); - Class propertyType = Util.getParameterizedType(property.getGenericType()); - Field[] declaredFields = propertyType.getDeclaredFields(); - boolean hasDynamicFields = Arrays.stream(declaredFields) - .anyMatch(field -> { - JsonProperty jsonProperty = field.getAnnotation(JsonProperty.class); - if (Objects.isNull(jsonProperty)) return false; - else return jsonProperty.value().equals("dynamicFields"); - }); - if (hasDynamicFields) { - if (value instanceof List) { - List> objects = (List>) value; - List> newObjects = new ArrayList<>(); - for (Map object : objects) { - Map clonedMap = new HashMap<>(object); - Object dynamicFields = clonedMap.get("dynamicFields"); - String followUps = (String)clonedMap.get("followUps"); - String dynamicFieldsValueAsJson = Objects.isNull(dynamicFields) - ? "" - : objectMapper.writeValueAsString(dynamicFields); - Object convertedFollowUps = Objects.isNull(followUps) - ? Collections.emptyList() - : objectMapper.readValue(followUps, new TypeReference>>() { - }); - clonedMap.put("dynamicFields", dynamicFieldsValueAsJson); - clonedMap.put("followUps", convertedFollowUps); - newObjects.add(clonedMap); - } - if (!newObjects.isEmpty()) - clonedProperty.put(key, newObjects); - } else { - Map object = (Map) value; - Map clonedObject = new HashMap<>(object); - Object dynamicFields = clonedObject.get("dynamicFields"); + Map dsmLevel = (Map) sourceMap.get(DSM); + ObjectMapper objectMapper = ObjectMapperSingleton.instance(); + if (Objects.isNull(dsmLevel)) { + return elasticSearchParticipantDto; + } + + Map clonedProperty = new HashMap<>(); + for (Map.Entry entry: dsmLevel.entrySet()) { + String key = entry.getKey(); + Object value = entry.getValue(); + try { + boolean hasDynamicFields = hasDynamicFields(key); + if (hasDynamicFields) { + if (value instanceof List) { + List> objects = (List>) value; + List> newObjects = new ArrayList<>(); + for (Map object : objects) { + Map clonedMap = new HashMap<>(object); + Object dynamicFields = clonedMap.get(ESObjectConstants.DYNAMIC_FIELDS); + String followUps = (String)clonedMap.get(ESObjectConstants.FOLLOW_UPS); String dynamicFieldsValueAsJson = Objects.isNull(dynamicFields) - ? "" + ? StringUtils.EMPTY : objectMapper.writeValueAsString(dynamicFields); - clonedObject.put("dynamicFields", dynamicFieldsValueAsJson); - clonedProperty.put(key, clonedObject); + Object convertedFollowUps = Objects.isNull(followUps) + ? Collections.emptyList() + : objectMapper.readValue(followUps, new TypeReference>>() { + }); + clonedMap.put(ESObjectConstants.DYNAMIC_FIELDS, dynamicFieldsValueAsJson); + clonedMap.put(ESObjectConstants.FOLLOW_UPS, convertedFollowUps); + newObjects.add(clonedMap); } + if (!newObjects.isEmpty()) + clonedProperty.put(key, newObjects); + } else { + Map object = (Map) value; + Map clonedObject = new HashMap<>(object); + Object dynamicFields = clonedObject.get(ESObjectConstants.DYNAMIC_FIELDS); + String dynamicFieldsValueAsJson = Objects.isNull(dynamicFields) + ? StringUtils.EMPTY + : objectMapper.writeValueAsString(dynamicFields); + clonedObject.put(ESObjectConstants.DYNAMIC_FIELDS, dynamicFieldsValueAsJson); + clonedProperty.put(key, clonedObject); } - } catch (NoSuchFieldException | ClassNotFoundException | IOException e) { - throw new RuntimeException(e); } + } catch (IOException e) { + throw new RuntimeException(e); } - if (!clonedProperty.isEmpty()) { - dsmLevel.putAll(clonedProperty); - } + } + if (!clonedProperty.isEmpty()) { + dsmLevel.putAll(clonedProperty); } elasticSearchParticipantDto = objectMapper.convertValue(sourceMap, ElasticSearchParticipantDto.class); return elasticSearchParticipantDto; } + private static boolean hasDynamicFields(String outerProperty) { + try { + Field property = ESDsm.class.getDeclaredField(outerProperty); + Class propertyType = Util.getParameterizedType(property.getGenericType()); + Field[] declaredFields = propertyType.getDeclaredFields(); + return Arrays.stream(declaredFields).anyMatch(ElasticSearch::isDynamicField); + } catch (NoSuchFieldException | ClassNotFoundException e) { + throw new RuntimeException(e); + } + } + + private static boolean isDynamicField(Field field) { + JsonProperty jsonProperty = field.getAnnotation(JsonProperty.class); + if (Objects.isNull(jsonProperty)) return false; + else return jsonProperty.value().equals(ESObjectConstants.DYNAMIC_FIELDS); + } + public List parseSourceMaps(SearchHit[] searchHits) { if (Objects.isNull(searchHits)) return Collections.emptyList(); List result = new ArrayList<>(); From 490078abcd70c64158198bb85257b2bf74cf5f3f Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 7 Dec 2021 17:32:54 +0400 Subject: [PATCH 202/405] DDP-7121 | continue refactoring of deserialize method of ElasticSearch --- .../model/elastic/search/ElasticSearch.java | 112 ++++++++++-------- 1 file changed, 61 insertions(+), 51 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java index 6fc8c72c9..d5d6cb615 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java @@ -13,6 +13,7 @@ import java.util.stream.Collectors; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; @@ -70,65 +71,74 @@ public long getTotalCount() { public static Optional parseSourceMap(Map sourceMap) { if (sourceMap == null) return Optional.of(new ElasticSearchParticipantDto.Builder().build()); - ElasticSearchParticipantDto elasticSearchParticipantDto = deserialize(sourceMap); - return Optional.of(elasticSearchParticipantDto); + return deserialize(sourceMap); } - private static ElasticSearchParticipantDto deserialize(Map sourceMap) { - ElasticSearchParticipantDto elasticSearchParticipantDto = null; + private static Optional deserialize(Map sourceMap) { + Optional elasticSearchParticipantDto = Optional.empty(); Map dsmLevel = (Map) sourceMap.get(DSM); - ObjectMapper objectMapper = ObjectMapperSingleton.instance(); - if (Objects.isNull(dsmLevel)) { - return elasticSearchParticipantDto; - } - Map clonedProperty = new HashMap<>(); + if (Objects.isNull(dsmLevel)) return elasticSearchParticipantDto; + + Map updatedPropertySourceMap = updatePropertySourceMapIfSpecialCases(dsmLevel); + if (!updatedPropertySourceMap.isEmpty()) dsmLevel.putAll(updatedPropertySourceMap); + + return Optional.of(ObjectMapperSingleton.instance().convertValue(sourceMap, ElasticSearchParticipantDto.class)); + } + + private static Map updatePropertySourceMapIfSpecialCases(Map dsmLevel) { + Map updatedPropertySourceMap = new HashMap<>(); for (Map.Entry entry: dsmLevel.entrySet()) { - String key = entry.getKey(); - Object value = entry.getValue(); - try { - boolean hasDynamicFields = hasDynamicFields(key); - if (hasDynamicFields) { - if (value instanceof List) { - List> objects = (List>) value; - List> newObjects = new ArrayList<>(); - for (Map object : objects) { - Map clonedMap = new HashMap<>(object); - Object dynamicFields = clonedMap.get(ESObjectConstants.DYNAMIC_FIELDS); - String followUps = (String)clonedMap.get(ESObjectConstants.FOLLOW_UPS); - String dynamicFieldsValueAsJson = Objects.isNull(dynamicFields) - ? StringUtils.EMPTY - : objectMapper.writeValueAsString(dynamicFields); - Object convertedFollowUps = Objects.isNull(followUps) - ? Collections.emptyList() - : objectMapper.readValue(followUps, new TypeReference>>() { - }); - clonedMap.put(ESObjectConstants.DYNAMIC_FIELDS, dynamicFieldsValueAsJson); - clonedMap.put(ESObjectConstants.FOLLOW_UPS, convertedFollowUps); - newObjects.add(clonedMap); - } - if (!newObjects.isEmpty()) - clonedProperty.put(key, newObjects); - } else { - Map object = (Map) value; - Map clonedObject = new HashMap<>(object); - Object dynamicFields = clonedObject.get(ESObjectConstants.DYNAMIC_FIELDS); - String dynamicFieldsValueAsJson = Objects.isNull(dynamicFields) - ? StringUtils.EMPTY - : objectMapper.writeValueAsString(dynamicFields); - clonedObject.put(ESObjectConstants.DYNAMIC_FIELDS, dynamicFieldsValueAsJson); - clonedProperty.put(key, clonedObject); - } - } - } catch (IOException e) { - throw new RuntimeException(e); + String outerProperty = entry.getKey(); + Object outerPropertyValue = entry.getValue(); + if (!hasDynamicFields(outerProperty)) continue; + if (outerPropertyValue instanceof List) { + List> outerPropertyValues = (List>) outerPropertyValue; + List> updatedOuterPropertyValues = handleSpecialCases(outerPropertyValues); + if (!updatedOuterPropertyValues.isEmpty()) + updatedPropertySourceMap.put(outerProperty, updatedOuterPropertyValues); + } else { + Map singleOuterPropertyValue = (Map) outerPropertyValue; + Map updatedSingleOuterPropertyValue = new HashMap<>(singleOuterPropertyValue); + updatedSingleOuterPropertyValue.put(ESObjectConstants.DYNAMIC_FIELDS, getDynamicFieldsValueAsJson(updatedSingleOuterPropertyValue)); + updatedPropertySourceMap.put(outerProperty, updatedSingleOuterPropertyValue); } } - if (!clonedProperty.isEmpty()) { - dsmLevel.putAll(clonedProperty); + return updatedPropertySourceMap; + } + + private static List> handleSpecialCases(List> outerPropertyValues) { + List> updatedOuterPropertyValues = new ArrayList<>(); + for (Map object : outerPropertyValues) { + Map clonedMap = new HashMap<>(object); + clonedMap.put(ESObjectConstants.DYNAMIC_FIELDS, getDynamicFieldsValueAsJson(clonedMap)); + clonedMap.put(ESObjectConstants.FOLLOW_UPS, convertFollowUpsJsonToList(clonedMap)); + updatedOuterPropertyValues.add(clonedMap); + } + return updatedOuterPropertyValues; + } + + private static List> convertFollowUpsJsonToList(Map clonedMap) { + String followUps = (String) clonedMap.get(ESObjectConstants.FOLLOW_UPS); + try { + return Objects.isNull(followUps) + ? Collections.emptyList() + : ObjectMapperSingleton.instance().readValue(followUps, new TypeReference>>() { + }); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + } + + private static String getDynamicFieldsValueAsJson(Map clonedMap) { + Object dynamicFields = clonedMap.get(ESObjectConstants.DYNAMIC_FIELDS); + try { + return Objects.isNull(dynamicFields) + ? StringUtils.EMPTY + : ObjectMapperSingleton.instance().writeValueAsString(dynamicFields); + } catch (JsonProcessingException jpe) { + throw new RuntimeException(jpe); } - elasticSearchParticipantDto = objectMapper.convertValue(sourceMap, ElasticSearchParticipantDto.class); - return elasticSearchParticipantDto; } private static boolean hasDynamicFields(String outerProperty) { From a03923f128e0de3d020b8950d9d57d192247633e Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 7 Dec 2021 17:55:48 +0400 Subject: [PATCH 203/405] DDP-7121 | finish fixing bugs for migration script and patch v1.0 --- .../model/elastic/search/Deserializer.java | 8 ++ .../model/elastic/search/ElasticSearch.java | 96 +---------------- .../elastic/search/SourceMapDeserializer.java | 101 ++++++++++++++++++ 3 files changed, 110 insertions(+), 95 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/search/Deserializer.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/Deserializer.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/Deserializer.java new file mode 100644 index 000000000..789a71e10 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/Deserializer.java @@ -0,0 +1,8 @@ +package org.broadinstitute.dsm.model.elastic.search; + +import java.util.Map; +import java.util.Optional; + +public interface Deserializer { + Optional deserialize(Map sourceMap); +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java index d5d6cb615..ee3af7436 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java @@ -1,7 +1,6 @@ package org.broadinstitute.dsm.model.elastic.search; import java.io.IOException; -import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -12,18 +11,11 @@ import java.util.Optional; import java.util.stream.Collectors; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; import lombok.Setter; import org.apache.commons.lang3.StringUtils; -import org.broadinstitute.dsm.model.elastic.ESDsm; import org.broadinstitute.dsm.model.elastic.Util; -import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; -import org.broadinstitute.dsm.util.ObjectMapperSingleton; import org.broadinstitute.dsm.util.ParticipantUtil; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; @@ -40,8 +32,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.broadinstitute.dsm.statics.ESObjectConstants.DSM; - @Setter public class ElasticSearch implements ElasticSearchable { @@ -71,91 +61,7 @@ public long getTotalCount() { public static Optional parseSourceMap(Map sourceMap) { if (sourceMap == null) return Optional.of(new ElasticSearchParticipantDto.Builder().build()); - return deserialize(sourceMap); - } - - private static Optional deserialize(Map sourceMap) { - Optional elasticSearchParticipantDto = Optional.empty(); - Map dsmLevel = (Map) sourceMap.get(DSM); - - if (Objects.isNull(dsmLevel)) return elasticSearchParticipantDto; - - Map updatedPropertySourceMap = updatePropertySourceMapIfSpecialCases(dsmLevel); - if (!updatedPropertySourceMap.isEmpty()) dsmLevel.putAll(updatedPropertySourceMap); - - return Optional.of(ObjectMapperSingleton.instance().convertValue(sourceMap, ElasticSearchParticipantDto.class)); - } - - private static Map updatePropertySourceMapIfSpecialCases(Map dsmLevel) { - Map updatedPropertySourceMap = new HashMap<>(); - for (Map.Entry entry: dsmLevel.entrySet()) { - String outerProperty = entry.getKey(); - Object outerPropertyValue = entry.getValue(); - if (!hasDynamicFields(outerProperty)) continue; - if (outerPropertyValue instanceof List) { - List> outerPropertyValues = (List>) outerPropertyValue; - List> updatedOuterPropertyValues = handleSpecialCases(outerPropertyValues); - if (!updatedOuterPropertyValues.isEmpty()) - updatedPropertySourceMap.put(outerProperty, updatedOuterPropertyValues); - } else { - Map singleOuterPropertyValue = (Map) outerPropertyValue; - Map updatedSingleOuterPropertyValue = new HashMap<>(singleOuterPropertyValue); - updatedSingleOuterPropertyValue.put(ESObjectConstants.DYNAMIC_FIELDS, getDynamicFieldsValueAsJson(updatedSingleOuterPropertyValue)); - updatedPropertySourceMap.put(outerProperty, updatedSingleOuterPropertyValue); - } - } - return updatedPropertySourceMap; - } - - private static List> handleSpecialCases(List> outerPropertyValues) { - List> updatedOuterPropertyValues = new ArrayList<>(); - for (Map object : outerPropertyValues) { - Map clonedMap = new HashMap<>(object); - clonedMap.put(ESObjectConstants.DYNAMIC_FIELDS, getDynamicFieldsValueAsJson(clonedMap)); - clonedMap.put(ESObjectConstants.FOLLOW_UPS, convertFollowUpsJsonToList(clonedMap)); - updatedOuterPropertyValues.add(clonedMap); - } - return updatedOuterPropertyValues; - } - - private static List> convertFollowUpsJsonToList(Map clonedMap) { - String followUps = (String) clonedMap.get(ESObjectConstants.FOLLOW_UPS); - try { - return Objects.isNull(followUps) - ? Collections.emptyList() - : ObjectMapperSingleton.instance().readValue(followUps, new TypeReference>>() { - }); - } catch (IOException ioe) { - throw new RuntimeException(ioe); - } - } - - private static String getDynamicFieldsValueAsJson(Map clonedMap) { - Object dynamicFields = clonedMap.get(ESObjectConstants.DYNAMIC_FIELDS); - try { - return Objects.isNull(dynamicFields) - ? StringUtils.EMPTY - : ObjectMapperSingleton.instance().writeValueAsString(dynamicFields); - } catch (JsonProcessingException jpe) { - throw new RuntimeException(jpe); - } - } - - private static boolean hasDynamicFields(String outerProperty) { - try { - Field property = ESDsm.class.getDeclaredField(outerProperty); - Class propertyType = Util.getParameterizedType(property.getGenericType()); - Field[] declaredFields = propertyType.getDeclaredFields(); - return Arrays.stream(declaredFields).anyMatch(ElasticSearch::isDynamicField); - } catch (NoSuchFieldException | ClassNotFoundException e) { - throw new RuntimeException(e); - } - } - - private static boolean isDynamicField(Field field) { - JsonProperty jsonProperty = field.getAnnotation(JsonProperty.class); - if (Objects.isNull(jsonProperty)) return false; - else return jsonProperty.value().equals(ESObjectConstants.DYNAMIC_FIELDS); + return new SourceMapDeserializer().deserialize(sourceMap); } public List parseSourceMaps(SearchHit[] searchHits) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java new file mode 100644 index 000000000..c462f457c --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java @@ -0,0 +1,101 @@ +package org.broadinstitute.dsm.model.elastic.search; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import org.apache.commons.lang3.StringUtils; +import org.broadinstitute.dsm.model.elastic.ESDsm; +import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.statics.ESObjectConstants; +import org.broadinstitute.dsm.util.ObjectMapperSingleton; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.*; + +public class SourceMapDeserializer implements Deserializer { + + public Optional deserialize(Map sourceMap) { + Optional elasticSearchParticipantDto = Optional.empty(); + Map dsmLevel = (Map) sourceMap.get(ESObjectConstants.DSM); + + if (Objects.isNull(dsmLevel)) return elasticSearchParticipantDto; + + Map updatedPropertySourceMap = updatePropertySourceMapIfSpecialCases(dsmLevel); + if (!updatedPropertySourceMap.isEmpty()) dsmLevel.putAll(updatedPropertySourceMap); + + return Optional.of(ObjectMapperSingleton.instance().convertValue(sourceMap, ElasticSearchParticipantDto.class)); + } + + private Map updatePropertySourceMapIfSpecialCases(Map dsmLevel) { + Map updatedPropertySourceMap = new HashMap(); + for (Map.Entry entry : dsmLevel.entrySet()) { + String outerProperty = entry.getKey(); + Object outerPropertyValue = entry.getValue(); + if (!hasDynamicFields(outerProperty)) continue; + if (outerPropertyValue instanceof List) { + List> outerPropertyValues = (List>) outerPropertyValue; + List> updatedOuterPropertyValues = handleSpecialCases(outerPropertyValues); + if (!updatedOuterPropertyValues.isEmpty()) + updatedPropertySourceMap.put(outerProperty, updatedOuterPropertyValues); + } else { + Map singleOuterPropertyValue = (Map) outerPropertyValue; + Map updatedSingleOuterPropertyValue = new HashMap(singleOuterPropertyValue); + updatedSingleOuterPropertyValue.put(ESObjectConstants.DYNAMIC_FIELDS, getDynamicFieldsValueAsJson(updatedSingleOuterPropertyValue)); + updatedPropertySourceMap.put(outerProperty, updatedSingleOuterPropertyValue); + } + } + return updatedPropertySourceMap; + } + + private List> handleSpecialCases(List> outerPropertyValues) { + List> updatedOuterPropertyValues = new ArrayList>(); + for (Map object : outerPropertyValues) { + Map clonedMap = new HashMap(object); + clonedMap.put(ESObjectConstants.DYNAMIC_FIELDS, getDynamicFieldsValueAsJson(clonedMap)); + clonedMap.put(ESObjectConstants.FOLLOW_UPS, convertFollowUpsJsonToList(clonedMap)); + updatedOuterPropertyValues.add(clonedMap); + } + return updatedOuterPropertyValues; + } + + private List> convertFollowUpsJsonToList(Map clonedMap) { + String followUps = (String) clonedMap.get(ESObjectConstants.FOLLOW_UPS); + try { + return Objects.isNull(followUps) + ? Collections.emptyList() + : ObjectMapperSingleton.instance().readValue(followUps, new TypeReference>>() { + }); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + } + + private String getDynamicFieldsValueAsJson(Map clonedMap) { + Object dynamicFields = clonedMap.get(ESObjectConstants.DYNAMIC_FIELDS); + try { + return Objects.isNull(dynamicFields) + ? StringUtils.EMPTY + : ObjectMapperSingleton.instance().writeValueAsString(dynamicFields); + } catch (JsonProcessingException jpe) { + throw new RuntimeException(jpe); + } + } + + private boolean hasDynamicFields(String outerProperty) { + try { + Field property = ESDsm.class.getDeclaredField(outerProperty); + Class propertyType = Util.getParameterizedType(property.getGenericType()); + Field[] declaredFields = propertyType.getDeclaredFields(); + return Arrays.stream(declaredFields).anyMatch(this::isDynamicField); + } catch (NoSuchFieldException | ClassNotFoundException e) { + throw new RuntimeException(e); + } + } + + private boolean isDynamicField(Field field) { + JsonProperty jsonProperty = field.getAnnotation(JsonProperty.class); + if (Objects.isNull(jsonProperty)) return false; + else return jsonProperty.value().equals(ESObjectConstants.DYNAMIC_FIELDS); + } +} \ No newline at end of file From 687cc06537353d2e203b4e1a98b208ffecb28b84 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 8 Dec 2021 10:26:42 +0400 Subject: [PATCH 204/405] DDP-7121 | start refactoring of participant wrapper for participant list --- .../ddp/participant/ParticipantDataDto.java | 20 +++++++ .../dsm/model/elastic/ESDsm.java | 3 +- .../model/participant/ParticipantWrapper.java | 58 +++++++++++++++++++ 3 files changed, 80 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java b/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java index 83ee7537f..39cdc81fb 100644 --- a/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java +++ b/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java @@ -1,11 +1,17 @@ package org.broadinstitute.dsm.db.dto.ddp.participant; +import java.io.IOException; import java.lang.reflect.Type; import java.util.HashMap; import java.util.Map; import java.util.Optional; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; +import com.google.gson.annotations.SerializedName; import com.google.gson.reflect.TypeToken; import lombok.Setter; import org.apache.commons.lang3.StringUtils; @@ -13,12 +19,14 @@ import org.broadinstitute.dsm.db.structure.ColumnName; import org.broadinstitute.dsm.db.structure.TableName; import org.broadinstitute.dsm.statics.DBConstants; +import org.broadinstitute.dsm.util.ObjectMapperSingleton; @TableName( name = DBConstants.DDP_PARTICIPANT_DATA, alias = DBConstants.DDP_PARTICIPANT_DATA_ALIAS, primaryKey = DBConstants.PARTICIPANT_DATA_ID, columnPrefix = "") +@JsonIgnoreProperties(ignoreUnknown = true) @Setter public class ParticipantDataDto { @@ -37,8 +45,20 @@ public class ParticipantDataDto { private String fieldTypeId; @ColumnName(ParticipantDataDao.DATA) + @JsonProperty("dynamicFields") private String data; + @JsonProperty("dynamicFields") + public Map getDynamicFields() { + try { + return ObjectMapperSingleton.instance().readValue(data, new TypeReference>() {}); + } catch (IOException e) { + return Map.of(); + } + } + + public ParticipantDataDto() {} + private long lastChanged; private String changedBy; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java index 23e362b3f..1ea5ec81c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java @@ -5,6 +5,7 @@ import com.google.gson.annotations.SerializedName; import lombok.Getter; import org.broadinstitute.dsm.db.*; +import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; import org.broadinstitute.dsm.statics.ESObjectConstants; @Getter @@ -44,7 +45,7 @@ public class ESDsm { List oncHistoryDetail; @SerializedName(ESObjectConstants.PARTICIPANT_DATA) - List participantData; + List participantData; @SerializedName(ESObjectConstants.PARTICIPANT) Participant participant; diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java index d49ecf32b..271d8e40d 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java @@ -9,6 +9,7 @@ import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; +import org.broadinstitute.dsm.model.elastic.ESDsm; import org.broadinstitute.dsm.model.elastic.ESProfile; import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; @@ -170,6 +171,19 @@ private void fetchAndPrepareData(DDPInstance ddpInstance) { ddpInstance.getParticipantIndexES(), participantWrapperPayload.getFrom(), participantWrapperPayload.getTo()); +// participants = getParticipantsFromEsData(); +// if (ddpInstance.isHasRole()) { +// medicalRecords = getMedicalRecordsFromEsData(); +// oncHistoryDetails = getOncHistoryDetailsFromEsData(); +// } +// if (DDPInstanceDao.getRole(ddpInstance.getName(), DBConstants.KIT_REQUEST_ACTIVATED)) { +// kitRequests = getKitRequestsFromEsData(); +// } +// abstractionActivities = +// AbstractionActivity.getAllAbstractionActivityByParticipantIds(ddpInstance.getName(), participantIds); +// abstractionSummary = AbstractionFinal.getAbstractionFinalByParticipantIds(ddpInstance.getName(), participantIds); +// proxiesByParticipantIds = getProxiesWithParticipantIdsFromElasticList(ddpInstance.getUsersIndexES(), esData.getEsParticipants()); +// participantData = getParticipantDataFromEsData(); List participantIds = getParticipantIdsFromElasticList(esData.getEsParticipants()); participants = Participant.getParticipantsByIds(ddpInstance.getName(), participantIds); if (ddpInstance.isHasRole()) { @@ -186,6 +200,50 @@ private void fetchAndPrepareData(DDPInstance ddpInstance) { participantData = new ParticipantDataDao().getParticipantDataByParticipantIds(participantIds); } + private Map> getParticipantDataFromEsData() { + Map> participantDataByParticipantId = new HashMap<>(); + for (ElasticSearchParticipantDto elasticSearchParticipantDto: esData.getEsParticipants()) { + String participantId = elasticSearchParticipantDto.getParticipantId(); + elasticSearchParticipantDto.getDsm().ifPresent(esDsm -> participantDataByParticipantId.put(participantId, esDsm.getParticipantData())); + } + return participantDataByParticipantId; + } + + private Map> getKitRequestsFromEsData() { + Map> kitRequestsByParticipantId = new HashMap<>(); + for (ElasticSearchParticipantDto elasticSearchParticipantDto: esData.getEsParticipants()) { + String participantId = elasticSearchParticipantDto.getParticipantId(); + elasticSearchParticipantDto.getDsm().ifPresent(esDsm -> kitRequestsByParticipantId.put(participantId, esDsm.getKitRequestShipping())); + } + return kitRequestsByParticipantId; + } + + private Map> getOncHistoryDetailsFromEsData() { + Map> oncHistoryDetailsByParticipantId = new HashMap<>(); + for (ElasticSearchParticipantDto elasticSearchParticipantDto: esData.getEsParticipants()) { + String participantId = elasticSearchParticipantDto.getParticipantId(); + elasticSearchParticipantDto.getDsm().ifPresent(esDsm -> oncHistoryDetailsByParticipantId.put(participantId, esDsm.getOncHistoryDetail())); + } + return oncHistoryDetailsByParticipantId; + } + + private Map> getMedicalRecordsFromEsData() { + Map> medicalRecordsByParticipantId = new HashMap<>(); + for (ElasticSearchParticipantDto elasticSearchParticipantDto: esData.getEsParticipants()) { + String participantId = elasticSearchParticipantDto.getParticipantId(); + elasticSearchParticipantDto.getDsm().ifPresent(esDsm -> medicalRecordsByParticipantId.put(participantId, esDsm.getMedicalRecord())); + } + return medicalRecordsByParticipantId; + } + + private List getParticipantsFromEsData() { + List participants = new ArrayList<>(); + for (ElasticSearchParticipantDto elasticSearchParticipantDto: esData.getEsParticipants()) { + elasticSearchParticipantDto.getDsm().ifPresent(esDsm -> participants.add(esDsm.getParticipant())); + } + return participants; + } + Map> getProxiesWithParticipantIdsFromElasticList(String esUsersIndex, List elasticSearchParticipantDtos) { Map> proxiesIdsFromElasticList = getProxiesIdsFromElasticList(elasticSearchParticipantDtos); return getProxiesWithParticipantIdsByProxiesIds(esUsersIndex, proxiesIdsFromElasticList); From 252edd1da013f8116c359a8439cebfb03d6e9104 Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 8 Dec 2021 11:08:45 +0400 Subject: [PATCH 205/405] DDP-7121 | fix key for medicalRecord, start implementing the parsing mechanism for dynamic fields written in pascal case --- .../export/process/CollectionProcessorFactory.java | 2 +- .../model/elastic/search/SourceMapDeserializer.java | 12 ++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorFactory.java index 86b7c569b..310ec1541 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorFactory.java @@ -9,7 +9,7 @@ public class CollectionProcessorFactory implements ProcessorFactory { public BaseProcessor make(BaseGenerator.PropertyInfo propertyInfo) { BaseProcessor processor = new CollectionProcessor(); if (propertyInfo.isCollection()) { - if (ESObjectConstants.MEDICAL_RECORDS.equals(propertyInfo.getPropertyName())) { + if (ESObjectConstants.MEDICAL_RECORD.equals(propertyInfo.getPropertyName())) { processor = new MedicalRecordCollectionProcessor(); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java index c462f457c..24b635575 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java @@ -12,6 +12,7 @@ import java.io.IOException; import java.lang.reflect.Field; import java.util.*; +import java.util.concurrent.ConcurrentHashMap; public class SourceMapDeserializer implements Deserializer { @@ -40,7 +41,7 @@ private Map updatePropertySourceMapIfSpecialCases(Map singleOuterPropertyValue = (Map) outerPropertyValue; - Map updatedSingleOuterPropertyValue = new HashMap(singleOuterPropertyValue); + Map updatedSingleOuterPropertyValue = new HashMap<>(singleOuterPropertyValue); updatedSingleOuterPropertyValue.put(ESObjectConstants.DYNAMIC_FIELDS, getDynamicFieldsValueAsJson(updatedSingleOuterPropertyValue)); updatedPropertySourceMap.put(outerProperty, updatedSingleOuterPropertyValue); } @@ -49,9 +50,9 @@ private Map updatePropertySourceMapIfSpecialCases(Map> handleSpecialCases(List> outerPropertyValues) { - List> updatedOuterPropertyValues = new ArrayList>(); + List> updatedOuterPropertyValues = new ArrayList<>(); for (Map object : outerPropertyValues) { - Map clonedMap = new HashMap(object); + Map clonedMap = new HashMap<>(object); clonedMap.put(ESObjectConstants.DYNAMIC_FIELDS, getDynamicFieldsValueAsJson(clonedMap)); clonedMap.put(ESObjectConstants.FOLLOW_UPS, convertFollowUpsJsonToList(clonedMap)); updatedOuterPropertyValues.add(clonedMap); @@ -72,7 +73,10 @@ private List> convertFollowUpsJsonToList(Map } private String getDynamicFieldsValueAsJson(Map clonedMap) { - Object dynamicFields = clonedMap.get(ESObjectConstants.DYNAMIC_FIELDS); + Map dynamicFields = new ConcurrentHashMap<>((Map) clonedMap.get(ESObjectConstants.DYNAMIC_FIELDS)); + for (Map.Entry key: dynamicFields.entrySet()) { + + } try { return Objects.isNull(dynamicFields) ? StringUtils.EMPTY From 832243a251d48c44c4ef846ded1fd114c68a28f4 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 8 Dec 2021 11:44:52 +0400 Subject: [PATCH 206/405] DDP-7121 | add camelCaseToPascalSnakeCase converter for participant data --- .../dsm/model/elastic/Util.java | 11 ++++++++-- .../elastic/search/SourceMapDeserializer.java | 16 +++++++++----- .../dsm/model/elastic/UtilTest.java | 12 ++++------ .../search/SourceMapDeserializerTest.java | 22 +++++++++++++++++++ 4 files changed, 45 insertions(+), 16 deletions(-) create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializerTest.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index ce2b10cc7..0e7a4adf3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -43,6 +43,7 @@ public class Util { public static final String UNDERSCORE_SEPARATOR = "_"; public static final String DOC = "_doc"; private static final Pattern CAMEL_CASE_REGEX = Pattern.compile("(([a-z])+([A-z]))*"); + private static final Pattern UPPER_CASE_REGEX = Pattern.compile("(?=\\p{Upper})"); public static final Gson GSON = new Gson(); public static String getQueryTypeFromId(String id) { @@ -189,8 +190,6 @@ public static Class getParameterizedType(Type genericType) throws ClassNotFou String typeAsString = genericType.toString(); String[] types = typeAsString.contains("<") ? typeAsString.split("<") : typeAsString.split("\\[L"); if (types.length < 2) { -// class NullClass {} -// return NullClass.class; return (Class) genericType; } String parameterizedType = types[1]; @@ -199,6 +198,14 @@ public static Class getParameterizedType(Type genericType) throws ClassNotFou return Class.forName(parameterizedType); } + public static String camelCaseToPascalSnakeCase(String camelCase) { + String[] words = camelCase.split(UPPER_CASE_REGEX.toString()); + String pascalSnakeCase = Arrays.stream(words) + .map(String::toUpperCase) + .collect(Collectors.joining(UNDERSCORE_SEPARATOR)); + return pascalSnakeCase; + } + public static class Constants { public static final String PROFILE = "profile"; public static final String PROFILE_HRUID = PROFILE + ".hruid"; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java index 24b635575..67266bc76 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java @@ -16,6 +16,8 @@ public class SourceMapDeserializer implements Deserializer { + private String outerProperty; + public Optional deserialize(Map sourceMap) { Optional elasticSearchParticipantDto = Optional.empty(); Map dsmLevel = (Map) sourceMap.get(ESObjectConstants.DSM); @@ -31,12 +33,12 @@ public Optional deserialize(Map sou private Map updatePropertySourceMapIfSpecialCases(Map dsmLevel) { Map updatedPropertySourceMap = new HashMap(); for (Map.Entry entry : dsmLevel.entrySet()) { - String outerProperty = entry.getKey(); + outerProperty = entry.getKey(); Object outerPropertyValue = entry.getValue(); if (!hasDynamicFields(outerProperty)) continue; if (outerPropertyValue instanceof List) { List> outerPropertyValues = (List>) outerPropertyValue; - List> updatedOuterPropertyValues = handleSpecialCases(outerPropertyValues); + List> updatedOuterPropertyValues = handleSpecialCases(outerPropertyValues);; if (!updatedOuterPropertyValues.isEmpty()) updatedPropertySourceMap.put(outerProperty, updatedOuterPropertyValues); } else { @@ -60,7 +62,7 @@ private List> handleSpecialCases(List> o return updatedOuterPropertyValues; } - private List> convertFollowUpsJsonToList(Map clonedMap) { + List> convertFollowUpsJsonToList(Map clonedMap) { String followUps = (String) clonedMap.get(ESObjectConstants.FOLLOW_UPS); try { return Objects.isNull(followUps) @@ -74,11 +76,13 @@ private List> convertFollowUpsJsonToList(Map private String getDynamicFieldsValueAsJson(Map clonedMap) { Map dynamicFields = new ConcurrentHashMap<>((Map) clonedMap.get(ESObjectConstants.DYNAMIC_FIELDS)); - for (Map.Entry key: dynamicFields.entrySet()) { - + if (ESObjectConstants.PARTICIPANT_DATA.equals(outerProperty)) { + for (Map.Entry entry: dynamicFields.entrySet()) { + dynamicFields.put(Util.camelCaseToPascalSnakeCase(entry.getKey()), entry.getValue()); + } } try { - return Objects.isNull(dynamicFields) + return dynamicFields.isEmpty() ? StringUtils.EMPTY : ObjectMapperSingleton.instance().writeValueAsString(dynamicFields); } catch (JsonProcessingException jpe) { diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java index 967cf4286..650417a4d 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java @@ -113,13 +113,9 @@ class MockClass { } @Test - public void go() { - - String s = "{followUps: \"[{\"fReceived\":\"2021-11-18\",\"fRequest1\":\"2021-11-18\"},{\"fReceived\":\"2021-11-18\",\"fRequest1\":\"2021-11-18\"}]\"}"; - - MedicalRecord medicalRecord = GSON.fromJson(s, MedicalRecord.class); - - System.out.println(); - + public void camelCaseToPascalSnakeCase() { + String camelCase = "registrationType"; + String pascalSnakeCase = Util.camelCaseToPascalSnakeCase(camelCase); + assertEquals("REGISTRATION_TYPE", pascalSnakeCase); } } \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializerTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializerTest.java new file mode 100644 index 000000000..c3417f018 --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializerTest.java @@ -0,0 +1,22 @@ +package org.broadinstitute.dsm.model.elastic.search; + +import static org.junit.Assert.*; + +import java.util.Map; + +import org.broadinstitute.dsm.statics.ESObjectConstants; +import org.junit.Test; + +public class SourceMapDeserializerTest { + + @Test + public void convertFollowUpsJsonToList() { + Map.of( + "REGISTRATION_TYPE", "Self", + "REGISTRATION_STATUS", "Registered" + ); + Map outerProperties = Map.of( + ESObjectConstants.DYNAMIC_FIELDS, + ) + } +} \ No newline at end of file From d39f425a08404660ed3f47a2b66b541f4568cc12 Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 8 Dec 2021 11:53:49 +0400 Subject: [PATCH 207/405] DDP-7121 | finish parsing camel case to pascal snake case --- .../elastic/search/SourceMapDeserializer.java | 6 ++--- .../search/SourceMapDeserializerTest.java | 22 ++++++++++++++----- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java index 67266bc76..11f4953f8 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java @@ -16,7 +16,7 @@ public class SourceMapDeserializer implements Deserializer { - private String outerProperty; + String outerProperty; public Optional deserialize(Map sourceMap) { Optional elasticSearchParticipantDto = Optional.empty(); @@ -62,7 +62,7 @@ private List> handleSpecialCases(List> o return updatedOuterPropertyValues; } - List> convertFollowUpsJsonToList(Map clonedMap) { + private List> convertFollowUpsJsonToList(Map clonedMap) { String followUps = (String) clonedMap.get(ESObjectConstants.FOLLOW_UPS); try { return Objects.isNull(followUps) @@ -74,7 +74,7 @@ List> convertFollowUpsJsonToList(Map clonedM } } - private String getDynamicFieldsValueAsJson(Map clonedMap) { + String getDynamicFieldsValueAsJson(Map clonedMap) { Map dynamicFields = new ConcurrentHashMap<>((Map) clonedMap.get(ESObjectConstants.DYNAMIC_FIELDS)); if (ESObjectConstants.PARTICIPANT_DATA.equals(outerProperty)) { for (Map.Entry entry: dynamicFields.entrySet()) { diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializerTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializerTest.java index c3417f018..8edd3437f 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializerTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializerTest.java @@ -2,21 +2,31 @@ import static org.junit.Assert.*; +import java.io.IOException; import java.util.Map; import org.broadinstitute.dsm.statics.ESObjectConstants; +import org.broadinstitute.dsm.util.ObjectMapperSingleton; +import org.junit.Assert; import org.junit.Test; public class SourceMapDeserializerTest { @Test public void convertFollowUpsJsonToList() { - Map.of( - "REGISTRATION_TYPE", "Self", - "REGISTRATION_STATUS", "Registered" + Map dynamicFields = Map.of( + "registrationType", "Self", + "registrationStatus", "Registered" ); - Map outerProperties = Map.of( - ESObjectConstants.DYNAMIC_FIELDS, - ) + Map outerProperties = Map.of(ESObjectConstants.DYNAMIC_FIELDS, dynamicFields); + SourceMapDeserializer sourceMapDeserializer = new SourceMapDeserializer(); + sourceMapDeserializer.outerProperty = ESObjectConstants.PARTICIPANT_DATA; + try { + Map dynamicFieldsValueAsJson = ObjectMapperSingleton.instance().readValue(sourceMapDeserializer.getDynamicFieldsValueAsJson(outerProperties), Map.class); + Assert.assertEquals(dynamicFieldsValueAsJson.get("REGISTRATION_TYPE"), "Self"); + Assert.assertEquals(dynamicFieldsValueAsJson.get("REGISTRATION_STATUS"), "Registered"); + } catch (IOException e) { + Assert.fail(); + } } } \ No newline at end of file From a6574a5fb9ae26ec9cac553179a840ed8a78e6e6 Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 8 Dec 2021 12:07:49 +0400 Subject: [PATCH 208/405] DDP-7121 | return empty map if there is no "dynamicFields" --- .../dsm/model/elastic/search/SourceMapDeserializer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java index 11f4953f8..d97974900 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java @@ -75,7 +75,7 @@ private List> convertFollowUpsJsonToList(Map } String getDynamicFieldsValueAsJson(Map clonedMap) { - Map dynamicFields = new ConcurrentHashMap<>((Map) clonedMap.get(ESObjectConstants.DYNAMIC_FIELDS)); + Map dynamicFields = new ConcurrentHashMap<>(clonedMap.get(ESObjectConstants.DYNAMIC_FIELDS) == null ? Map.of() : (Map) clonedMap.get(ESObjectConstants.DYNAMIC_FIELDS)); if (ESObjectConstants.PARTICIPANT_DATA.equals(outerProperty)) { for (Map.Entry entry: dynamicFields.entrySet()) { dynamicFields.put(Util.camelCaseToPascalSnakeCase(entry.getKey()), entry.getValue()); From 4c7bcfb3d9a383b25dcf8249353439737bc70e8f Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 9 Dec 2021 10:15:36 +0400 Subject: [PATCH 209/405] DDP-7121 | continue refactoring participant wrapper --- .../dsm/model/elastic/ESDsm.java | 27 +++++++++++- .../elastic/search/SourceMapDeserializer.java | 6 ++- .../model/participant/ParticipantWrapper.java | 43 ++++++++++--------- .../dsm/model/elastic/UtilTest.java | 3 ++ .../search/SourceMapDeserializerTest.java | 7 ++- 5 files changed, 62 insertions(+), 24 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java index 1ea5ec81c..4c7633fdb 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java @@ -1,5 +1,6 @@ package org.broadinstitute.dsm.model.elastic; +import java.util.Collections; import java.util.List; import com.google.gson.annotations.SerializedName; @@ -36,7 +37,7 @@ public class ESDsm { Object diagnosisYear; @SerializedName(ESObjectConstants.TISSUE_RECORDS) - List tissue; // todo: change to Tissue + List tissue; @SerializedName(ESObjectConstants.MEDICAL_RECORDS) List medicalRecord; @@ -54,4 +55,28 @@ public class ESDsm { OncHistory oncHistory; + public List getTissue() { + if (tissue == null) tissue = Collections.emptyList(); + return tissue; + } + + public List getMedicalRecord() { + if (medicalRecord == null) medicalRecord = Collections.emptyList(); + return medicalRecord; + } + + public List getOncHistoryDetail() { + if (oncHistoryDetail == null) oncHistoryDetail = Collections.emptyList(); + return oncHistoryDetail; + } + + public List getParticipantData() { + if (participantData == null) participantData = Collections.emptyList(); + return participantData; + } + + public List getKitRequestShipping() { + if (kitRequestShipping == null) kitRequestShipping = Collections.emptyList(); + return kitRequestShipping; + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java index d97974900..2312c7da5 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java @@ -75,11 +75,13 @@ private List> convertFollowUpsJsonToList(Map } String getDynamicFieldsValueAsJson(Map clonedMap) { - Map dynamicFields = new ConcurrentHashMap<>(clonedMap.get(ESObjectConstants.DYNAMIC_FIELDS) == null ? Map.of() : (Map) clonedMap.get(ESObjectConstants.DYNAMIC_FIELDS)); + Map dynamicFields = clonedMap.get(ESObjectConstants.DYNAMIC_FIELDS) == null ? Map.of() : (Map) clonedMap.get(ESObjectConstants.DYNAMIC_FIELDS); if (ESObjectConstants.PARTICIPANT_DATA.equals(outerProperty)) { + Map updatedParticipantDataDynamicFields = new HashMap<>(); for (Map.Entry entry: dynamicFields.entrySet()) { - dynamicFields.put(Util.camelCaseToPascalSnakeCase(entry.getKey()), entry.getValue()); + updatedParticipantDataDynamicFields.put(Util.camelCaseToPascalSnakeCase(entry.getKey()), entry.getValue()); } + dynamicFields = updatedParticipantDataDynamicFields; } try { return dynamicFields.isEmpty() diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java index 271d8e40d..1724e13b1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java @@ -171,33 +171,33 @@ private void fetchAndPrepareData(DDPInstance ddpInstance) { ddpInstance.getParticipantIndexES(), participantWrapperPayload.getFrom(), participantWrapperPayload.getTo()); -// participants = getParticipantsFromEsData(); + participants = getParticipantsFromEsData(); + if (ddpInstance.isHasRole()) { + medicalRecords = getMedicalRecordsFromEsData(); + oncHistoryDetails = getOncHistoryDetailsFromEsData(); + } + if (DDPInstanceDao.getRole(ddpInstance.getName(), DBConstants.KIT_REQUEST_ACTIVATED)) { + kitRequests = getKitRequestsFromEsData(); + } +// abstractionActivities = +// AbstractionActivity.getAllAbstractionActivityByParticipantIds(ddpInstance.getName(), participantIds); +// abstractionSummary = AbstractionFinal.getAbstractionFinalByParticipantIds(ddpInstance.getName(), participantIds); + proxiesByParticipantIds = getProxiesWithParticipantIdsFromElasticList(ddpInstance.getUsersIndexES(), esData.getEsParticipants()); + participantData = getParticipantDataFromEsData(); +// List participantIds = getParticipantIdsFromElasticList(esData.getEsParticipants()); +// participants = Participant.getParticipantsByIds(ddpInstance.getName(), participantIds); // if (ddpInstance.isHasRole()) { -// medicalRecords = getMedicalRecordsFromEsData(); -// oncHistoryDetails = getOncHistoryDetailsFromEsData(); +// medicalRecords = MedicalRecord.getMedicalRecordsByParticipantIds(ddpInstance.getName(), participantIds); +// oncHistoryDetails = OncHistoryDetail.getOncHistoryDetailsByParticipantIds(ddpInstance.getName(), participantIds); // } // if (DDPInstanceDao.getRole(ddpInstance.getName(), DBConstants.KIT_REQUEST_ACTIVATED)) { -// kitRequests = getKitRequestsFromEsData(); +// kitRequests = KitRequestShipping.getKitRequestsByParticipantIds(ddpInstance, participantIds); // } // abstractionActivities = // AbstractionActivity.getAllAbstractionActivityByParticipantIds(ddpInstance.getName(), participantIds); // abstractionSummary = AbstractionFinal.getAbstractionFinalByParticipantIds(ddpInstance.getName(), participantIds); // proxiesByParticipantIds = getProxiesWithParticipantIdsFromElasticList(ddpInstance.getUsersIndexES(), esData.getEsParticipants()); -// participantData = getParticipantDataFromEsData(); - List participantIds = getParticipantIdsFromElasticList(esData.getEsParticipants()); - participants = Participant.getParticipantsByIds(ddpInstance.getName(), participantIds); - if (ddpInstance.isHasRole()) { - medicalRecords = MedicalRecord.getMedicalRecordsByParticipantIds(ddpInstance.getName(), participantIds); - oncHistoryDetails = OncHistoryDetail.getOncHistoryDetailsByParticipantIds(ddpInstance.getName(), participantIds); - } - if (DDPInstanceDao.getRole(ddpInstance.getName(), DBConstants.KIT_REQUEST_ACTIVATED)) { - kitRequests = KitRequestShipping.getKitRequestsByParticipantIds(ddpInstance, participantIds); - } - abstractionActivities = - AbstractionActivity.getAllAbstractionActivityByParticipantIds(ddpInstance.getName(), participantIds); - abstractionSummary = AbstractionFinal.getAbstractionFinalByParticipantIds(ddpInstance.getName(), participantIds); - proxiesByParticipantIds = getProxiesWithParticipantIdsFromElasticList(ddpInstance.getUsersIndexES(), esData.getEsParticipants()); - participantData = new ParticipantDataDao().getParticipantDataByParticipantIds(participantIds); +// participantData = new ParticipantDataDao().getParticipantDataByParticipantIds(participantIds); } private Map> getParticipantDataFromEsData() { @@ -239,7 +239,10 @@ private Map> getMedicalRecordsFromEsData() { private List getParticipantsFromEsData() { List participants = new ArrayList<>(); for (ElasticSearchParticipantDto elasticSearchParticipantDto: esData.getEsParticipants()) { - elasticSearchParticipantDto.getDsm().ifPresent(esDsm -> participants.add(esDsm.getParticipant())); + elasticSearchParticipantDto.getDsm().ifPresent(esDsm -> { + Participant participant = esDsm.getParticipant(); + if (Objects.nonNull(participant)) participants.add(participant); + }); } return participants; } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java index 650417a4d..c8153607f 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java @@ -115,7 +115,10 @@ class MockClass { @Test public void camelCaseToPascalSnakeCase() { String camelCase = "registrationType"; + String camelCase2 = "test"; String pascalSnakeCase = Util.camelCaseToPascalSnakeCase(camelCase); + String pascalSnakeCase2 = Util.camelCaseToPascalSnakeCase(camelCase2); assertEquals("REGISTRATION_TYPE", pascalSnakeCase); + assertEquals("TEST", pascalSnakeCase2); } } \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializerTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializerTest.java index 8edd3437f..4b00faa35 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializerTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializerTest.java @@ -18,13 +18,18 @@ public void convertFollowUpsJsonToList() { "registrationType", "Self", "registrationStatus", "Registered" ); - Map outerProperties = Map.of(ESObjectConstants.DYNAMIC_FIELDS, dynamicFields); + + Map outerProperties = Map.of( + "ddpInstanceId", 12, + ESObjectConstants.DYNAMIC_FIELDS, dynamicFields + ); SourceMapDeserializer sourceMapDeserializer = new SourceMapDeserializer(); sourceMapDeserializer.outerProperty = ESObjectConstants.PARTICIPANT_DATA; try { Map dynamicFieldsValueAsJson = ObjectMapperSingleton.instance().readValue(sourceMapDeserializer.getDynamicFieldsValueAsJson(outerProperties), Map.class); Assert.assertEquals(dynamicFieldsValueAsJson.get("REGISTRATION_TYPE"), "Self"); Assert.assertEquals(dynamicFieldsValueAsJson.get("REGISTRATION_STATUS"), "Registered"); + Assert.assertEquals(12, outerProperties.get("ddpInstanceId")); } catch (IOException e) { Assert.fail(); } From fde797214c8bcf0bfbd7a9642dbdf3b67affb63b Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 9 Dec 2021 10:44:57 +0400 Subject: [PATCH 210/405] DDP-7121 | add Optional in deserializer --- src/main/java/org/broadinstitute/dsm/model/elastic/Util.java | 4 ++-- .../dsm/model/elastic/search/ElasticSearch.java | 4 +++- .../elastic/migration/DynamicFieldsMappingMigratorTest.java | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 0e7a4adf3..7f919fb1c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -25,6 +25,7 @@ import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.statics.ESObjectConstants; +import org.broadinstitute.dsm.util.ObjectMapperSingleton; import org.broadinstitute.dsm.util.ParticipantUtil; import org.broadinstitute.dsm.util.PatchUtil; @@ -154,9 +155,8 @@ private static Map dynamicFieldsSpecialCase(Object fieldValue) { Map dynamicMap = new HashMap<>(); if (isJsonInString(fieldValue)) { String strValue = (String) fieldValue; - ObjectMapper objectMapper = new ObjectMapper(); try { - dynamicMap = objectMapper.readValue(strValue, Map.class); + dynamicMap = ObjectMapperSingleton.instance().readValue(strValue, Map.class); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java index ee3af7436..0a850f409 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java @@ -61,7 +61,9 @@ public long getTotalCount() { public static Optional parseSourceMap(Map sourceMap) { if (sourceMap == null) return Optional.of(new ElasticSearchParticipantDto.Builder().build()); - return new SourceMapDeserializer().deserialize(sourceMap); + return new SourceMapDeserializer().deserialize(sourceMap).isPresent() + ? new SourceMapDeserializer().deserialize(sourceMap) + : Optional.of(new ElasticSearchParticipantDto.Builder().build()); } public List parseSourceMaps(SearchHit[] searchHits) { diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java index 30aab1fe9..53e7a9928 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java @@ -17,8 +17,8 @@ public static void setUp() { @Test public void testExport() { - final String index = "participants_structured.rgp.rgp"; - final String study = "rgp"; + final String index = "participants_structured.cmi.angio"; + final String study = "angio"; List exportables = Arrays.asList( //DynamicFieldsMappingMigrator should be first in the list to make sure that mapping will be exported for first new DynamicFieldsMappingMigrator(index, study), From 420045b0d7a5568f43715196890d22594bd34915 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 9 Dec 2021 11:15:21 +0400 Subject: [PATCH 211/405] DDP-7121 | add conditions to check whether dynamic field and follow ups exist in source map --- .../dsm/model/elastic/search/SourceMapDeserializer.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java index 2312c7da5..0c5c467e7 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java @@ -55,8 +55,12 @@ private List> handleSpecialCases(List> o List> updatedOuterPropertyValues = new ArrayList<>(); for (Map object : outerPropertyValues) { Map clonedMap = new HashMap<>(object); - clonedMap.put(ESObjectConstants.DYNAMIC_FIELDS, getDynamicFieldsValueAsJson(clonedMap)); - clonedMap.put(ESObjectConstants.FOLLOW_UPS, convertFollowUpsJsonToList(clonedMap)); + if (object.containsKey(ESObjectConstants.DYNAMIC_FIELDS)) { + clonedMap.put(ESObjectConstants.DYNAMIC_FIELDS, getDynamicFieldsValueAsJson(clonedMap)); + } + if (object.containsKey(ESObjectConstants.FOLLOW_UPS)) { + clonedMap.put(ESObjectConstants.FOLLOW_UPS, convertFollowUpsJsonToList(clonedMap)); + } updatedOuterPropertyValues.add(clonedMap); } return updatedOuterPropertyValues; From b4207c3a5dd182b8b68fb46029eb5695f9a5e26d Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 9 Dec 2021 15:45:41 +0400 Subject: [PATCH 212/405] DDP-7121 | start implementing SingleProcessor for the data which is represented by one-to-one relationships (OncHistory, Participant) --- .../model/elastic/export/ExportFacade.java | 7 ++ .../elastic/export/process/BaseProcessor.java | 49 +++++++++++++ .../export/process/CollectionProcessor.java | 72 +++++-------------- .../elastic/export/process/Processor.java | 5 +- .../export/process/SingleProcessor.java | 44 ++++++++++++ .../model/elastic/search/ElasticSearch.java | 5 +- .../elastic/search/SourceMapDeserializer.java | 9 +-- 7 files changed, 128 insertions(+), 63 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/process/SingleProcessor.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 135acf3c2..001e54be5 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -17,6 +17,7 @@ import org.broadinstitute.dsm.model.elastic.export.parse.ValueParser; import org.broadinstitute.dsm.model.elastic.export.process.BaseProcessor; import org.broadinstitute.dsm.model.elastic.export.process.CollectionProcessorFactory; +import org.broadinstitute.dsm.model.elastic.export.process.SingleProcessor; import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchable; @@ -90,6 +91,12 @@ private Map processData(ESDsm esDsm) { new HashMap<>(Map.of(propertyInfo.getPropertyName(), processedData)))); } + } else { + processor = new SingleProcessor(); + processor.setEsDsm(esDsm); + processor.setPropertyName(propertyInfo.getPropertyName()); + processor.setRecordId(exportFacadePayload.getRecordId()); + processor.setCollector(generator); } logger.info("Returning processed ES participant data"); return dataToReturn; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/BaseProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/BaseProcessor.java index bc58116d3..068c7c9a8 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/BaseProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/BaseProcessor.java @@ -1,17 +1,26 @@ package org.broadinstitute.dsm.model.elastic.export.process; +import java.lang.reflect.Field; +import java.util.Map; import java.util.Objects; +import java.util.Optional; +import org.apache.commons.lang3.StringUtils; +import org.broadinstitute.dsm.db.structure.TableName; import org.broadinstitute.dsm.model.elastic.ESDsm; import org.broadinstitute.dsm.model.elastic.export.generate.Collector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public abstract class BaseProcessor implements Processor { + protected static final Logger logger = LoggerFactory.getLogger(CollectionProcessor.class); protected ESDsm esDsm; protected String propertyName; protected int recordId; protected Collector collector; + protected String primaryKey; public BaseProcessor(ESDsm esDsm, String propertyName, int recordId, Collector collector) { this.esDsm = Objects.requireNonNull(esDsm); @@ -39,4 +48,44 @@ public void setRecordId(int recordId) { public void setCollector(Collector collector) { this.collector = collector; } + + protected String getPrimaryKey(Object obj) { + TableName tableName = obj.getClass().getAnnotation(TableName.class); + return tableName != null ? tableName.primaryKey() : StringUtils.EMPTY; + } + + protected void updateExistingRecord(Map eachRecord) { + logger.info("Updating existing record"); + Optional> maybeEndResult = collectEndResult(); + if (maybeEndResult.isPresent()) { + for (Map.Entry endResultEntry : maybeEndResult.get().entrySet()) { + String endResultEntryKey = endResultEntry.getKey(); + Object eachRecordValue = eachRecord.get(endResultEntryKey); + if (eachRecordValue instanceof Map) { + Map eachRecordEntryMap = (Map) eachRecordValue; + eachRecordEntryMap.putAll((Map) endResultEntry.getValue()); + } else { + eachRecord.putAll(maybeEndResult.get()); + } + } + } + } + + protected abstract Object extractDataByReflection(); + + protected Object getValueByField(Field field) { + try { + Object value = field.get(esDsm); + primaryKey = findPrimaryKeyOfObject(value); + return convertObjectToCollection(value); + } catch (IllegalAccessException iae) { + throw new RuntimeException("error occurred while attempting to get data from ESDsm", iae); + } + } + + protected abstract Object convertObjectToCollection(Object object); + + protected abstract String findPrimaryKeyOfObject(Object object); + + protected abstract Optional> collectEndResult(); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java index ae989ba98..16f978aa7 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java @@ -2,28 +2,17 @@ import java.lang.reflect.Field; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.function.Predicate; -import org.broadinstitute.dsm.db.structure.TableName; import org.broadinstitute.dsm.model.elastic.ESDsm; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.generate.Collector; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class CollectionProcessor extends BaseProcessor { - private static final Logger logger = LoggerFactory.getLogger(CollectionProcessor.class); - - private String primaryKey; - - private final Predicate isFieldMatchProperty = field -> propertyName.equals(field.getName()); - public CollectionProcessor(ESDsm esDsm, String propertyName, int recordId, Collector collector) { super(esDsm, propertyName, recordId, collector); } @@ -38,37 +27,31 @@ public List> process() { return updateIfExistsOrPut(fetchedRecords); } - private List> extractDataByReflection() { + @Override + protected List> extractDataByReflection() { logger.info("Extracting data by field from fetched ES data"); - Field[] declaredFields = esDsm.getClass().getDeclaredFields(); - List> fetchedRecords = Arrays.stream(declaredFields).filter(isFieldMatchProperty) - .findFirst() - .map(field -> { - field.setAccessible(true); - return getRecordsByField(field); - }) - .orElse(new ArrayList<>()); - return fetchedRecords; - } - - private List> getRecordsByField(Field field) { try { - List objectCollection = (List) field.get(esDsm); - primaryKey = findPrimaryKeyOfObject(objectCollection); - return Util.convertObjectListToMapList(objectCollection); - } catch (IllegalAccessException iae) { - throw new RuntimeException("error occurred while attempting to get data from ESDsm", iae); + Field declaredField = esDsm.getClass().getDeclaredField(propertyName); + declaredField.setAccessible(true); + return (List>) getValueByField(declaredField); + + } catch (NoSuchFieldException e) { + return new ArrayList<>(); } } - private String findPrimaryKeyOfObject(List objectCollection) { + @Override + protected Object convertObjectToCollection(Object object) { + return Util.convertObjectListToMapList(object); + } + + @Override + protected String findPrimaryKeyOfObject(Object object) { + List objectCollection = (List) object; if (Objects.isNull(objectCollection)) return ""; Optional maybeObject = objectCollection.stream().findFirst(); return maybeObject - .map(o -> { - TableName tableName = o.getClass().getAnnotation(TableName.class); - return tableName != null ? tableName.primaryKey() : ""; - }) + .map(this::getPrimaryKey) .orElse(""); } @@ -86,30 +69,13 @@ private boolean isExistingRecord(Map eachRecord) { return id == (double) recordId; } - private void updateExistingRecord(Map eachRecord) { - logger.info("Updating existing record"); - Optional> maybeEndResult = collectEndResult(); - if (maybeEndResult.isPresent()) { - for (Map.Entry endResultEntry : maybeEndResult.get().entrySet()) { - String endResultEntryKey = endResultEntry.getKey(); - Object eachRecordValue = eachRecord.get(endResultEntryKey); - if (eachRecordValue instanceof Map) { - Map eachRecordEntryMap = (Map) eachRecordValue; - eachRecordEntryMap.putAll((Map) endResultEntry.getValue()); - } else { - eachRecord.putAll(maybeEndResult.get()); - } - } - } - - } - private void addNewRecordTo(List> fetchedRecords) { logger.info("Adding new record"); collectEndResult().ifPresent(fetchedRecords::add); } - private Optional> collectEndResult() { + @Override + protected Optional> collectEndResult() { return ((List>) collector.collect()) .stream() .findFirst(); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/Processor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/Processor.java index 80edc8db6..181e434b4 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/Processor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/Processor.java @@ -1,8 +1,5 @@ package org.broadinstitute.dsm.model.elastic.export.process; -import java.util.List; -import java.util.Map; - public interface Processor { - List> process(); + Object process(); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/SingleProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/SingleProcessor.java new file mode 100644 index 000000000..7c3a46552 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/SingleProcessor.java @@ -0,0 +1,44 @@ +package org.broadinstitute.dsm.model.elastic.export.process; + +import java.lang.reflect.Field; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +public class SingleProcessor extends BaseProcessor { + + + @Override + public Map process() { + Map fetchedRecord = extractDataByReflection(); + + return null; + + } + + @Override + protected Map extractDataByReflection() { + logger.info("Extracting data by field from fetched ES data"); + return null; + } + + @Override + protected Map getValueByField(Field field) { + return null; + } + + @Override + protected Object convertObjectToCollection(Object object) { + return null; + } + + @Override + protected String findPrimaryKeyOfObject(Object objectCollection) { + return null; + } + + @Override + protected Optional> collectEndResult() { + return Optional.empty(); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java index 0a850f409..adb9f8ebf 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java @@ -61,8 +61,9 @@ public long getTotalCount() { public static Optional parseSourceMap(Map sourceMap) { if (sourceMap == null) return Optional.of(new ElasticSearchParticipantDto.Builder().build()); - return new SourceMapDeserializer().deserialize(sourceMap).isPresent() - ? new SourceMapDeserializer().deserialize(sourceMap) + Optional deserializedSourceMap = new SourceMapDeserializer().deserialize(sourceMap); + return deserializedSourceMap.isPresent() + ? deserializedSourceMap : Optional.of(new ElasticSearchParticipantDto.Builder().build()); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java index 0c5c467e7..4136e8624 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java @@ -19,10 +19,9 @@ public class SourceMapDeserializer implements Deserializer { String outerProperty; public Optional deserialize(Map sourceMap) { - Optional elasticSearchParticipantDto = Optional.empty(); Map dsmLevel = (Map) sourceMap.get(ESObjectConstants.DSM); - if (Objects.isNull(dsmLevel)) return elasticSearchParticipantDto; + if (Objects.isNull(dsmLevel)) return Optional.of(ObjectMapperSingleton.instance().convertValue(sourceMap, ElasticSearchParticipantDto.class)); Map updatedPropertySourceMap = updatePropertySourceMapIfSpecialCases(dsmLevel); if (!updatedPropertySourceMap.isEmpty()) dsmLevel.putAll(updatedPropertySourceMap); @@ -44,7 +43,9 @@ private Map updatePropertySourceMapIfSpecialCases(Map singleOuterPropertyValue = (Map) outerPropertyValue; Map updatedSingleOuterPropertyValue = new HashMap<>(singleOuterPropertyValue); - updatedSingleOuterPropertyValue.put(ESObjectConstants.DYNAMIC_FIELDS, getDynamicFieldsValueAsJson(updatedSingleOuterPropertyValue)); + if (singleOuterPropertyValue.containsKey(ESObjectConstants.DYNAMIC_FIELDS)) + updatedSingleOuterPropertyValue.put(ESObjectConstants.DYNAMIC_FIELDS, getDynamicFieldsValueAsJson(updatedSingleOuterPropertyValue)); + updatedPropertySourceMap.put(outerProperty, updatedSingleOuterPropertyValue); } } @@ -79,7 +80,7 @@ private List> convertFollowUpsJsonToList(Map } String getDynamicFieldsValueAsJson(Map clonedMap) { - Map dynamicFields = clonedMap.get(ESObjectConstants.DYNAMIC_FIELDS) == null ? Map.of() : (Map) clonedMap.get(ESObjectConstants.DYNAMIC_FIELDS); + Map dynamicFields = (Map) clonedMap.get(ESObjectConstants.DYNAMIC_FIELDS); if (ESObjectConstants.PARTICIPANT_DATA.equals(outerProperty)) { Map updatedParticipantDataDynamicFields = new HashMap<>(); for (Map.Entry entry: dynamicFields.entrySet()) { From 598727badfa5126d5fd0408c5810e3d2ddf2780e Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 9 Dec 2021 17:02:40 +0400 Subject: [PATCH 213/405] DDP-7121 | add methods to complete single processor --- .../dsm/model/elastic/Util.java | 6 +++- .../model/elastic/export/ExportFacade.java | 2 +- .../elastic/export/process/BaseProcessor.java | 17 +++++++++- .../export/process/CollectionProcessor.java | 20 +++--------- .../MedicalRecordCollectionProcessor.java | 3 +- .../export/process/SingleProcessor.java | 31 ++++++++----------- .../export/process/SingleProcessorTest.java | 17 ++++++++++ 7 files changed, 58 insertions(+), 38 deletions(-) create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/export/process/SingleProcessorTest.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 7f919fb1c..5d7a28f21 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -148,7 +148,11 @@ static Map convertToMap(String fieldName, Object fieldValue) { } public static List> convertObjectListToMapList(Object fieldValue) { - return new ObjectMapper().convertValue(fieldValue, new TypeReference>>() {}); + return ObjectMapperSingleton.instance().convertValue(fieldValue, new TypeReference>>() {}); + } + + public static Map convertObjectToMap(Object fieldValue) { + return ObjectMapperSingleton.instance().convertValue(fieldValue, new TypeReference>() {}); } private static Map dynamicFieldsSpecialCase(Object fieldValue) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 001e54be5..60f3149c8 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -85,7 +85,7 @@ private Map processData(ESDsm esDsm) { processor.setPropertyName(propertyInfo.getPropertyName()); processor.setRecordId(exportFacadePayload.getRecordId()); processor.setCollector(generator); - List> processedData = processor.process(); + List> processedData = (List>) processor.process(); if (!processedData.isEmpty()) { dataToReturn = new HashMap<>(Map.of(MappingGenerator.DSM_OBJECT, new HashMap<>(Map.of(propertyInfo.getPropertyName(), diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/BaseProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/BaseProcessor.java index 068c7c9a8..c594cc835 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/BaseProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/BaseProcessor.java @@ -1,6 +1,8 @@ package org.broadinstitute.dsm.model.elastic.export.process; import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -71,7 +73,18 @@ protected void updateExistingRecord(Map eachRecord) { } } - protected abstract Object extractDataByReflection(); + protected Object extractDataByReflection() { + logger.info("Extracting data by field from fetched ES data"); + try { + Field declaredField = esDsm.getClass().getDeclaredField(propertyName); + declaredField.setAccessible(true); + return getValueByField(declaredField); + } catch (NoSuchFieldException e) { + return this instanceof CollectionProcessor + ? new ArrayList<>() + : Map.of(); + } + }; protected Object getValueByField(Field field) { try { @@ -87,5 +100,7 @@ protected Object getValueByField(Field field) { protected abstract String findPrimaryKeyOfObject(Object object); + protected abstract Object updateIfExistsOrPut(Object value); + protected abstract Optional> collectEndResult(); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java index 16f978aa7..a0770ae35 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java @@ -1,7 +1,5 @@ package org.broadinstitute.dsm.model.elastic.export.process; -import java.lang.reflect.Field; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; @@ -23,22 +21,10 @@ public CollectionProcessor() { @Override public List> process() { - List> fetchedRecords = extractDataByReflection(); + List> fetchedRecords = (List>) extractDataByReflection(); return updateIfExistsOrPut(fetchedRecords); } - @Override - protected List> extractDataByReflection() { - logger.info("Extracting data by field from fetched ES data"); - try { - Field declaredField = esDsm.getClass().getDeclaredField(propertyName); - declaredField.setAccessible(true); - return (List>) getValueByField(declaredField); - - } catch (NoSuchFieldException e) { - return new ArrayList<>(); - } - } @Override protected Object convertObjectToCollection(Object object) { @@ -55,7 +41,9 @@ protected String findPrimaryKeyOfObject(Object object) { .orElse(""); } - protected List> updateIfExistsOrPut(List> fetchedRecords) { + @Override + protected List> updateIfExistsOrPut(Object value) { + List> fetchedRecords = (List>) value; fetchedRecords.stream() .filter(this::isExistingRecord) .findFirst() diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/MedicalRecordCollectionProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/MedicalRecordCollectionProcessor.java index d0b754557..c52d65ffc 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/MedicalRecordCollectionProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/MedicalRecordCollectionProcessor.java @@ -14,7 +14,8 @@ public MedicalRecordCollectionProcessor() { } @Override - protected List> updateIfExistsOrPut(List> fetchedRecords) { + protected List> updateIfExistsOrPut(Object value) { + List> fetchedRecords = (List>) value; for (Map medicalRecord: fetchedRecords) { Object followUps = medicalRecord.get(ESObjectConstants.FOLLOW_UPS); if (!Objects.isNull(followUps)) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/SingleProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/SingleProcessor.java index 7c3a46552..33ed566b8 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/SingleProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/SingleProcessor.java @@ -1,44 +1,39 @@ package org.broadinstitute.dsm.model.elastic.export.process; -import java.lang.reflect.Field; -import java.util.List; import java.util.Map; import java.util.Optional; +import org.broadinstitute.dsm.model.elastic.Util; + public class SingleProcessor extends BaseProcessor { @Override public Map process() { - Map fetchedRecord = extractDataByReflection(); - - return null; + Map fetchedRecord = (Map) extractDataByReflection(); + return updateIfExistsOrPut(fetchedRecord); } @Override - protected Map extractDataByReflection() { - logger.info("Extracting data by field from fetched ES data"); - return null; - } - - @Override - protected Map getValueByField(Field field) { - return null; + protected Map convertObjectToCollection(Object object) { + return Util.convertObjectToMap(object); } @Override - protected Object convertObjectToCollection(Object object) { - return null; + protected String findPrimaryKeyOfObject(Object object) { + return getPrimaryKey(object); } @Override - protected String findPrimaryKeyOfObject(Object objectCollection) { - return null; + protected Map updateIfExistsOrPut(Object value) { + Map record = (Map) value; + updateExistingRecord(record); + return record; } @Override protected Optional> collectEndResult() { - return Optional.empty(); + return Optional.ofNullable((Map) collector.collect()); } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/SingleProcessorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/SingleProcessorTest.java new file mode 100644 index 000000000..d6ca8d745 --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/SingleProcessorTest.java @@ -0,0 +1,17 @@ +package org.broadinstitute.dsm.model.elastic.export.process; + +import static org.junit.Assert.*; + +import java.util.Map; + +import org.junit.Test; + +public class SingleProcessorTest { + + @Test + public void process() { + Map.of( + + ); + } +} \ No newline at end of file From 68c776c66e6dad71bbeafabb3ab8134061ebf6de Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 9 Dec 2021 17:33:52 +0400 Subject: [PATCH 214/405] DDP-7121 | add @Setter ESDsm for tests, make unit tests pass --- .../dsm/model/elastic/ESDsm.java | 2 + .../process/CollectionProcessorTest.java | 2 +- .../export/process/SingleProcessorTest.java | 56 +++++++++++++++++-- 3 files changed, 53 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java index 4c7633fdb..d6988a0ab 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java @@ -5,11 +5,13 @@ import com.google.gson.annotations.SerializedName; import lombok.Getter; +import lombok.Setter; import org.broadinstitute.dsm.db.*; import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; import org.broadinstitute.dsm.statics.ESObjectConstants; @Getter +@Setter public class ESDsm { @SerializedName(ESObjectConstants.DATE_OF_MAJORITY) diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorTest.java index 3f5088673..8e6ffec82 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorTest.java @@ -38,7 +38,7 @@ public void testProcess() throws IOException { Processor collectionProcessor = new TestCollectionProcessor(esDsm, propertyName, generatorPayload, instance(generatorPayload, nameValue)); - List> updatedList = collectionProcessor.process(); + List> updatedList = (List>) collectionProcessor.process(); Map updatedObject = updatedList.get(0); Map oldObject = ((Map) ((List) objectMapper.readValue(json, Map.class).get(propertyName)).get(0)); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/SingleProcessorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/SingleProcessorTest.java index d6ca8d745..b5464a2a0 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/SingleProcessorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/SingleProcessorTest.java @@ -1,17 +1,61 @@ package org.broadinstitute.dsm.model.elastic.export.process; -import static org.junit.Assert.*; +import org.broadinstitute.dsm.db.Participant; +import org.broadinstitute.dsm.model.NameValue; +import org.broadinstitute.dsm.model.elastic.ESDsm; +import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; +import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorFactory; +import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; +import org.broadinstitute.dsm.model.elastic.export.generate.SourceGeneratorFactory; +import org.broadinstitute.dsm.model.elastic.export.parse.ValueParser; +import org.broadinstitute.dsm.statics.DBConstants; +import org.junit.Assert; +import org.junit.Test; import java.util.Map; -import org.junit.Test; - public class SingleProcessorTest { @Test - public void process() { - Map.of( + public void processExisting() { + BaseGenerator.PropertyInfo propertyInfo = Util.TABLE_ALIAS_MAPPINGS.get(DBConstants.DDP_PARTICIPANT_ALIAS); + ValueParser valueParser = new ValueParser(); + GeneratorFactory sourceGeneratorFactory = new SourceGeneratorFactory(); + BaseGenerator generator = sourceGeneratorFactory.make(propertyInfo); + generator.setParser(valueParser); + generator.setPayload(new GeneratorPayload(new NameValue("p.additionalValuesJson", "{\"key\":\"value\"}"), 0)); + + ESDsm esDsm = new ESDsm(); + esDsm.setParticipant(new Participant("2174", null, null, null, null, null, null, + null, null, null, false, false, "{\"key\": \"oldVal\"}", 12874512387612L )); + BaseProcessor processor = new SingleProcessor(); + processor.setEsDsm(esDsm); + processor.setPropertyName(propertyInfo.getPropertyName()); + processor.setRecordId(0); + processor.setCollector(generator); + Map processedData = (Map) processor.process(); + Assert.assertEquals("value", processedData.get("key")); + } + + @Test + public void processNew() { + BaseGenerator.PropertyInfo propertyInfo = Util.TABLE_ALIAS_MAPPINGS.get(DBConstants.DDP_PARTICIPANT_ALIAS); + ValueParser valueParser = new ValueParser(); + GeneratorFactory sourceGeneratorFactory = new SourceGeneratorFactory(); + BaseGenerator generator = sourceGeneratorFactory.make(propertyInfo); + generator.setParser(valueParser); + generator.setPayload(new GeneratorPayload(new NameValue("p.additionalValuesJson", "{\"key\":\"value\"}"), 0)); - ); + ESDsm esDsm = new ESDsm(); + esDsm.setParticipant(new Participant("2174", null, null, null, null, null, null, + null, null, null, false, false, "", 12874512387612L )); + BaseProcessor processor = new SingleProcessor(); + processor.setEsDsm(esDsm); + processor.setPropertyName(propertyInfo.getPropertyName()); + processor.setRecordId(0); + processor.setCollector(generator); + Map processedData = (Map) processor.process(); + Assert.assertEquals("value", processedData.get("key")); } } \ No newline at end of file From a109be8d3717763de28e2af8b9ee4885e7651fa6 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 10 Dec 2021 11:07:47 +0400 Subject: [PATCH 215/405] DDP-7121 | add null pointer exception handling, add null checks on processor --- .../broadinstitute/dsm/db/MedicalRecord.java | 5 +-- .../dsm/db/OncHistoryDetail.java | 5 +-- .../broadinstitute/dsm/db/Participant.java | 5 +-- .../dsm/db/ParticipantData.java | 5 +-- .../org/broadinstitute/dsm/db/Tissue.java | 5 +-- .../ddp/participant/ParticipantDataDto.java | 2 +- .../ddp/participant/ParticipantRecordDto.java | 5 +-- .../model/elastic/export/ExportFacade.java | 35 +++++++------------ .../export/generate/SourceGenerator.java | 3 +- .../elastic/export/process/BaseProcessor.java | 2 ++ .../export/process/ProcessorFactory.java | 2 +- ...Factory.java => ProcessorFactoryImpl.java} | 7 ++-- .../export/process/SingleProcessorTest.java | 5 +-- 13 files changed, 45 insertions(+), 41 deletions(-) rename src/main/java/org/broadinstitute/dsm/model/elastic/export/process/{CollectionProcessorFactory.java => ProcessorFactoryImpl.java} (71%) diff --git a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java index e0260e178..3be5357b2 100644 --- a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java +++ b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java @@ -33,6 +33,7 @@ import org.broadinstitute.dsm.statics.RoutePath; import org.broadinstitute.dsm.util.DBUtil; import org.broadinstitute.dsm.util.DDPRequestUtil; +import org.broadinstitute.dsm.util.ObjectMapperSingleton; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -344,8 +345,8 @@ public boolean isFollowUpRequired() { @JsonProperty("dynamicFields") public Map getDynamicFields() { try { - return new ObjectMapper().readValue(additionalValuesJson, new TypeReference>() {}); - } catch (IOException e) { + return ObjectMapperSingleton.instance().readValue(additionalValuesJson, new TypeReference>() {}); + } catch (IOException | NullPointerException e) { return Map.of(); } } diff --git a/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java b/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java index 0c883afab..e37414b33 100644 --- a/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java +++ b/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java @@ -18,6 +18,7 @@ import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.statics.QueryExtension; import org.broadinstitute.dsm.util.DBUtil; +import org.broadinstitute.dsm.util.ObjectMapperSingleton; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -172,8 +173,8 @@ public class OncHistoryDetail { @JsonProperty("dynamicFields") public Map getDynamicFields() { try { - return new ObjectMapper().readValue(additionalValuesJson, new TypeReference>() {}); - } catch (IOException e) { + return ObjectMapperSingleton.instance().readValue(additionalValuesJson, new TypeReference>() {}); + } catch (IOException | NullPointerException e) { return Map.of(); } } diff --git a/src/main/java/org/broadinstitute/dsm/db/Participant.java b/src/main/java/org/broadinstitute/dsm/db/Participant.java index c11c4da45..b4b5e7eee 100644 --- a/src/main/java/org/broadinstitute/dsm/db/Participant.java +++ b/src/main/java/org/broadinstitute/dsm/db/Participant.java @@ -14,6 +14,7 @@ import org.broadinstitute.dsm.db.structure.TableName; import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.util.DBUtil; +import org.broadinstitute.dsm.util.ObjectMapperSingleton; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -134,8 +135,8 @@ public class Participant { @JsonProperty("dynamicFields") public Map getDynamicFields() { try { - return new ObjectMapper().readValue(additionalValuesJson, new TypeReference>() {}); - } catch (IOException e) { + return ObjectMapperSingleton.instance().readValue(additionalValuesJson, new TypeReference>() {}); + } catch (IOException | NullPointerException e) { return Map.of(); } } diff --git a/src/main/java/org/broadinstitute/dsm/db/ParticipantData.java b/src/main/java/org/broadinstitute/dsm/db/ParticipantData.java index 29d410b66..976515c1a 100644 --- a/src/main/java/org/broadinstitute/dsm/db/ParticipantData.java +++ b/src/main/java/org/broadinstitute/dsm/db/ParticipantData.java @@ -13,6 +13,7 @@ import org.broadinstitute.dsm.db.structure.TableName; import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.util.DBUtil; +import org.broadinstitute.dsm.util.ObjectMapperSingleton; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,8 +66,8 @@ public class ParticipantData { @JsonProperty("dynamicFields") public Map getDynamicFields() { try { - return new ObjectMapper().readValue(data, new TypeReference>() {}); - } catch (IOException e) { + return ObjectMapperSingleton.instance().readValue(data, new TypeReference>() {}); + } catch (IOException | NullPointerException e) { return Map.of(); } } diff --git a/src/main/java/org/broadinstitute/dsm/db/Tissue.java b/src/main/java/org/broadinstitute/dsm/db/Tissue.java index 91c9b814e..86abb5aee 100644 --- a/src/main/java/org/broadinstitute/dsm/db/Tissue.java +++ b/src/main/java/org/broadinstitute/dsm/db/Tissue.java @@ -14,6 +14,7 @@ import org.broadinstitute.dsm.db.structure.SqlDateConverter; import org.broadinstitute.dsm.db.structure.TableName; import org.broadinstitute.dsm.statics.DBConstants; +import org.broadinstitute.dsm.util.ObjectMapperSingleton; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -117,8 +118,8 @@ public class Tissue { @JsonProperty("dynamicFields") public Map getDynamicFields() { try { - return new ObjectMapper().readValue(additionalValuesJson, new TypeReference>() {}); - } catch (IOException e) { + return ObjectMapperSingleton.instance().readValue(additionalValuesJson, new TypeReference>() {}); + } catch (IOException | NullPointerException e) { return Map.of(); } } diff --git a/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java b/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java index 39cdc81fb..1d43b23ee 100644 --- a/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java +++ b/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java @@ -52,7 +52,7 @@ public class ParticipantDataDto { public Map getDynamicFields() { try { return ObjectMapperSingleton.instance().readValue(data, new TypeReference>() {}); - } catch (IOException e) { + } catch (IOException | NullPointerException e) { return Map.of(); } } diff --git a/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantRecordDto.java b/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantRecordDto.java index 89eec91cb..0fc7c210d 100644 --- a/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantRecordDto.java +++ b/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantRecordDto.java @@ -15,6 +15,7 @@ import org.broadinstitute.dsm.db.structure.ColumnName; import org.broadinstitute.dsm.db.structure.TableName; import org.broadinstitute.dsm.statics.DBConstants; +import org.broadinstitute.dsm.util.ObjectMapperSingleton; @TableName( name = DBConstants.DDP_PARTICIPANT_RECORD, @@ -53,8 +54,8 @@ public class ParticipantRecordDto { @JsonProperty("dynamicFields") public Map getDynamicFields() { try { - return new ObjectMapper().readValue(additionalValuesJson, new TypeReference>() {}); - } catch (IOException e) { + return ObjectMapperSingleton.instance().readValue(additionalValuesJson, new TypeReference>() {}); + } catch (IOException | NullPointerException e) { return Map.of(); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 60f3149c8..2532d0dbd 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -16,7 +16,9 @@ import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; import org.broadinstitute.dsm.model.elastic.export.parse.ValueParser; import org.broadinstitute.dsm.model.elastic.export.process.BaseProcessor; -import org.broadinstitute.dsm.model.elastic.export.process.CollectionProcessorFactory; +import org.broadinstitute.dsm.model.elastic.export.process.Processor; +import org.broadinstitute.dsm.model.elastic.export.process.ProcessorFactory; +import org.broadinstitute.dsm.model.elastic.export.process.ProcessorFactoryImpl; import org.broadinstitute.dsm.model.elastic.export.process.SingleProcessor; import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; @@ -76,28 +78,17 @@ private Map processData(ESDsm esDsm) { generator = sourceGeneratorFactory.make(propertyInfo); generator.setParser(valueParser); generator.setPayload(exportFacadePayload.getGeneratorPayload()); - Map dataToReturn = generator.generate(); logger.info("Processing ES participant data"); - if (propertyInfo.isCollection()) { - CollectionProcessorFactory collectionProcessorFactory = new CollectionProcessorFactory(); - processor = collectionProcessorFactory.make(propertyInfo); - processor.setEsDsm(esDsm); - processor.setPropertyName(propertyInfo.getPropertyName()); - processor.setRecordId(exportFacadePayload.getRecordId()); - processor.setCollector(generator); - List> processedData = (List>) processor.process(); - if (!processedData.isEmpty()) { - dataToReturn = new HashMap<>(Map.of(MappingGenerator.DSM_OBJECT, - new HashMap<>(Map.of(propertyInfo.getPropertyName(), - processedData)))); - } - } else { - processor = new SingleProcessor(); - processor.setEsDsm(esDsm); - processor.setPropertyName(propertyInfo.getPropertyName()); - processor.setRecordId(exportFacadePayload.getRecordId()); - processor.setCollector(generator); - } + ProcessorFactory processorFactory = new ProcessorFactoryImpl(); + processor = processorFactory.make(propertyInfo); + processor.setEsDsm(esDsm); + processor.setPropertyName(propertyInfo.getPropertyName()); + processor.setRecordId(exportFacadePayload.getRecordId()); + processor.setCollector(generator); + Object processedData = processor.process(); + Map dataToReturn = new HashMap<>(Map.of(MappingGenerator.DSM_OBJECT, + new HashMap<>(Map.of(propertyInfo.getPropertyName(), + processedData)))); logger.info("Returning processed ES participant data"); return dataToReturn; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java index b409f040c..c94e35bcb 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java @@ -5,6 +5,7 @@ import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; +import org.broadinstitute.dsm.statics.ESObjectConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,7 +42,7 @@ protected Map parseJsonValuesToObject() { for (Map.Entry entry : dynamicFieldValues.entrySet()) { transformedMap.put(Util.underscoresToCamelCase(entry.getKey()), parser.parse(String.valueOf(entry.getValue()))); } - return transformedMap; + return new HashMap<>(Map.of(ESObjectConstants.DYNAMIC_FIELDS, transformedMap)); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/BaseProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/BaseProcessor.java index c594cc835..99b5acfee 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/BaseProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/BaseProcessor.java @@ -52,11 +52,13 @@ public void setCollector(Collector collector) { } protected String getPrimaryKey(Object obj) { + if (Objects.isNull(obj)) return StringUtils.EMPTY; TableName tableName = obj.getClass().getAnnotation(TableName.class); return tableName != null ? tableName.primaryKey() : StringUtils.EMPTY; } protected void updateExistingRecord(Map eachRecord) { + if (Objects.isNull(eachRecord)) return; logger.info("Updating existing record"); Optional> maybeEndResult = collectEndResult(); if (maybeEndResult.isPresent()) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/ProcessorFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/ProcessorFactory.java index 369b73cc5..125421025 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/ProcessorFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/ProcessorFactory.java @@ -3,5 +3,5 @@ import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; public interface ProcessorFactory { - Processor make(BaseGenerator.PropertyInfo propertyInfo); + BaseProcessor make(BaseGenerator.PropertyInfo propertyInfo); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/ProcessorFactoryImpl.java similarity index 71% rename from src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorFactory.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/export/process/ProcessorFactoryImpl.java index 310ec1541..43dd12178 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/ProcessorFactoryImpl.java @@ -3,15 +3,18 @@ import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; import org.broadinstitute.dsm.statics.ESObjectConstants; -public class CollectionProcessorFactory implements ProcessorFactory { +public class ProcessorFactoryImpl implements ProcessorFactory { @Override public BaseProcessor make(BaseGenerator.PropertyInfo propertyInfo) { - BaseProcessor processor = new CollectionProcessor(); + BaseProcessor processor; if (propertyInfo.isCollection()) { + processor = new CollectionProcessor(); if (ESObjectConstants.MEDICAL_RECORD.equals(propertyInfo.getPropertyName())) { processor = new MedicalRecordCollectionProcessor(); } + } else { + processor = new SingleProcessor(); } return processor; } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/SingleProcessorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/SingleProcessorTest.java index b5464a2a0..1c4dc8b06 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/SingleProcessorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/SingleProcessorTest.java @@ -10,6 +10,7 @@ import org.broadinstitute.dsm.model.elastic.export.generate.SourceGeneratorFactory; import org.broadinstitute.dsm.model.elastic.export.parse.ValueParser; import org.broadinstitute.dsm.statics.DBConstants; +import org.broadinstitute.dsm.statics.ESObjectConstants; import org.junit.Assert; import org.junit.Test; @@ -35,7 +36,7 @@ public void processExisting() { processor.setRecordId(0); processor.setCollector(generator); Map processedData = (Map) processor.process(); - Assert.assertEquals("value", processedData.get("key")); + Assert.assertEquals("value", ((Map)processedData.get(ESObjectConstants.DYNAMIC_FIELDS)).get("key")); } @Test @@ -56,6 +57,6 @@ public void processNew() { processor.setRecordId(0); processor.setCollector(generator); Map processedData = (Map) processor.process(); - Assert.assertEquals("value", processedData.get("key")); + Assert.assertEquals("value", ((Map)processedData.get(ESObjectConstants.DYNAMIC_FIELDS)).get("key")); } } \ No newline at end of file From f4f86938c84095d810e3a3393826e2d726a68d9e Mon Sep 17 00:00:00 2001 From: Nika Date: Fri, 10 Dec 2021 11:25:07 +0400 Subject: [PATCH 216/405] DDP-7121 | handle NullPointerExceptions, return empty HashMaps and Lists where necessary --- .../java/org/broadinstitute/dsm/model/elastic/Util.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 5d7a28f21..a42c6919c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -148,11 +148,15 @@ static Map convertToMap(String fieldName, Object fieldValue) { } public static List> convertObjectListToMapList(Object fieldValue) { - return ObjectMapperSingleton.instance().convertValue(fieldValue, new TypeReference>>() {}); + return Objects.isNull(fieldValue) + ? Collections.emptyList() + : ObjectMapperSingleton.instance().convertValue(fieldValue, new TypeReference>>() {}); } public static Map convertObjectToMap(Object fieldValue) { - return ObjectMapperSingleton.instance().convertValue(fieldValue, new TypeReference>() {}); + return Objects.isNull(fieldValue) + ? new HashMap<>() + : ObjectMapperSingleton.instance().convertValue(fieldValue, new TypeReference>() {}); } private static Map dynamicFieldsSpecialCase(Object fieldValue) { From 8a23eff67c1fe87ce5a0652546ab757ffa795b1d Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 13 Dec 2021 10:49:30 +0400 Subject: [PATCH 217/405] DDP-7121 | literal fixes --- .../ddp/participant/ParticipantRecordDto.java | 5 ----- .../export/generate/SingleSourceGenerator.java | 3 ++- .../export/generate/SourceGenerator.java | 2 +- .../dsm/model/patch/BasePatch.java | 18 +++++++++--------- 4 files changed, 12 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantRecordDto.java b/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantRecordDto.java index 0fc7c210d..203e24399 100644 --- a/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantRecordDto.java +++ b/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantRecordDto.java @@ -17,11 +17,6 @@ import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.util.ObjectMapperSingleton; -@TableName( - name = DBConstants.DDP_PARTICIPANT_RECORD, - alias = DBConstants.DDP_PARTICIPANT_RECORD_ALIAS, - primaryKey = DBConstants.PARTICIPANT_RECORD_ID, - columnPrefix = "") @JsonIgnoreProperties(ignoreUnknown = true) @JsonInclude(JsonInclude.Include.NON_NULL) public class ParticipantRecordDto { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleSourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleSourceGenerator.java index 2b1a974b1..ea31188b4 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleSourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SingleSourceGenerator.java @@ -5,6 +5,7 @@ import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; +import org.broadinstitute.dsm.statics.ESObjectConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -16,7 +17,7 @@ public SingleSourceGenerator() {} @Override public Object construct() { - return parseJsonValuesToObject(); + return new HashMap<>(Map.of(ESObjectConstants.DYNAMIC_FIELDS, parseJsonValuesToObject())); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java index c94e35bcb..d1e47529e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java @@ -42,7 +42,7 @@ protected Map parseJsonValuesToObject() { for (Map.Entry entry : dynamicFieldValues.entrySet()) { transformedMap.put(Util.underscoresToCamelCase(entry.getKey()), parser.parse(String.valueOf(entry.getValue()))); } - return new HashMap<>(Map.of(ESObjectConstants.DYNAMIC_FIELDS, transformedMap)); + return transformedMap; } } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index 979f626a4..1931d5b9d 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -204,19 +204,19 @@ else if (patch.getNameValue().getName().equals("m.faxSent3")) { nameValues.add(setAdditionalValue("m.faxConfirmed3", patch, patch.getNameValue().getValue())); } //tissue request workflow - else if (patch.getNameValue().getName().equals("oD.tFaxSent")) { - nameValues.add(setAdditionalValue("oD.tFaxSentBy", patch, patch.getUser())); - nameValues.add(setAdditionalValue("oD.tFaxConfirmed", patch, patch.getNameValue().getValue())); + else if (patch.getNameValue().getName().equals("oD.faxSent")) { + nameValues.add(setAdditionalValue("oD.faxSentBy", patch, patch.getUser())); + nameValues.add(setAdditionalValue("oD.faxConfirmed", patch, patch.getNameValue().getValue())); nameValues.add(setAdditionalValue("oD.request", patch, "sent")); } - else if (patch.getNameValue().getName().equals("oD.tFaxSent2")) { - nameValues.add(setAdditionalValue("oD.tFaxSent2By", patch, patch.getUser())); - nameValues.add(setAdditionalValue("oD.tFaxConfirmed2", patch, patch.getNameValue().getValue())); + else if (patch.getNameValue().getName().equals("oD.faxSent2")) { + nameValues.add(setAdditionalValue("oD.faxSent2By", patch, patch.getUser())); + nameValues.add(setAdditionalValue("oD.faxConfirmed2", patch, patch.getNameValue().getValue())); nameValues.add(setAdditionalValue("oD.request", patch, "sent")); } - else if (patch.getNameValue().getName().equals("oD.tFaxSent3")) { - nameValues.add(setAdditionalValue("oD.tFaxSent3By", patch, patch.getUser())); - nameValues.add(setAdditionalValue("oD.tFaxConfirmed3", patch, patch.getNameValue().getValue())); + else if (patch.getNameValue().getName().equals("oD.faxSent3")) { + nameValues.add(setAdditionalValue("oD.faxSent3By", patch, patch.getUser())); + nameValues.add(setAdditionalValue("oD.faxConfirmed3", patch, patch.getNameValue().getValue())); nameValues.add(setAdditionalValue("oD.request", patch, "sent")); } else if (patch.getNameValue().getName().equals("oD.tissueReceived")) { From ebc5b3dac1162daa674c2049cd953280c0ab47f5 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 13 Dec 2021 16:18:14 +0400 Subject: [PATCH 218/405] DDP-7121 | changes for tissue --- .../org/broadinstitute/dsm/db/Tissue.java | 32 +++++++++---------- .../dsm/model/elastic/Util.java | 2 +- .../elastic/export/process/BaseProcessor.java | 3 +- .../export/process/CollectionProcessor.java | 5 +-- 4 files changed, 22 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/Tissue.java b/src/main/java/org/broadinstitute/dsm/db/Tissue.java index 86abb5aee..44d406ddf 100644 --- a/src/main/java/org/broadinstitute/dsm/db/Tissue.java +++ b/src/main/java/org/broadinstitute/dsm/db/Tissue.java @@ -56,51 +56,51 @@ public class Tissue { private String tissueId; @ColumnName(DBConstants.ONC_HISTORY_DETAIL_ID) - private final String oncHistoryDetailId; + private String oncHistoryDetailId; @ColumnName (DBConstants.NOTES) - private final String notes; + private String notes; @ColumnName (DBConstants.COUNT_RECEIVED) - private final Integer countReceived; + private Integer countReceived; @ColumnName (DBConstants.TISSUE_TYPE) - private final String tissueType; + private String tissueType; @ColumnName (DBConstants.TISSUE_SITE) - private final String tissueSite; + private String tissueSite; @ColumnName (DBConstants.TUMOR_TYPE) - private final String tumorType; + private String tumorType; @ColumnName (DBConstants.H_E) - private final String hE; + private String hE; @ColumnName (DBConstants.PATHOLOGY_REPORT) - private final String pathologyReport; + private String pathologyReport; @ColumnName (DBConstants.COLLABORATOR_SAMPLE_ID) - private final String collaboratorSampleId; + private String collaboratorSampleId; @ColumnName (DBConstants.BLOCK_SENT) - private final String blockSent; + private String blockSent; @ColumnName (DBConstants.SHL_WORK_NUMBER) - private final String shlWorkNumber; + private String shlWorkNumber; @ColumnName (DBConstants.SCROLLS_RECEIVED) @DbDateConversion(SqlDateConverter.STRING_DAY) - private final String scrollsReceived; + private String scrollsReceived; @ColumnName (DBConstants.SK_ID) - private final String skId; + private String skId; @ColumnName (DBConstants.SM_ID) - private final String smId; + private String smId; @ColumnName (DBConstants.SENT_GP) @DbDateConversion(SqlDateConverter.STRING_DAY) - private final String sentGp; + private String sentGp; private String changedBy; @@ -152,7 +152,7 @@ public Map getDynamicFields() { @ColumnName (DBConstants.H_E_COUNT) private Integer hECount; - + public Tissue() {} public Tissue(String tissueId, String oncHistoryDetailId, String notes, Integer countReceived, String tissueType, String tissueSite, String tumorType, String hE, String pathologyReport, String collaboratorSampleId, diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index a42c6919c..b7b96e06e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -149,7 +149,7 @@ static Map convertToMap(String fieldName, Object fieldValue) { public static List> convertObjectListToMapList(Object fieldValue) { return Objects.isNull(fieldValue) - ? Collections.emptyList() + ? new ArrayList<>() : ObjectMapperSingleton.instance().convertValue(fieldValue, new TypeReference>>() {}); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/BaseProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/BaseProcessor.java index 99b5acfee..d14efd75b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/BaseProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/BaseProcessor.java @@ -2,6 +2,7 @@ import java.lang.reflect.Field; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -84,7 +85,7 @@ protected Object extractDataByReflection() { } catch (NoSuchFieldException e) { return this instanceof CollectionProcessor ? new ArrayList<>() - : Map.of(); + : new HashMap<>(); } }; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java index a0770ae35..25954910a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessor.java @@ -5,6 +5,7 @@ import java.util.Objects; import java.util.Optional; +import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.elastic.ESDsm; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.generate.Collector; @@ -34,11 +35,11 @@ protected Object convertObjectToCollection(Object object) { @Override protected String findPrimaryKeyOfObject(Object object) { List objectCollection = (List) object; - if (Objects.isNull(objectCollection)) return ""; + if (Objects.isNull(objectCollection)) return StringUtils.EMPTY; Optional maybeObject = objectCollection.stream().findFirst(); return maybeObject .map(this::getPrimaryKey) - .orElse(""); + .orElse(StringUtils.EMPTY); } @Override From d1654cde0f5b9678198f42c54468e10ed7ec6b1d Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 13 Dec 2021 18:13:47 +0400 Subject: [PATCH 219/405] DDP-7121 | add TissueSourceGenerator and change SourceGeneratorFactory --- .../org/broadinstitute/dsm/db/Tissue.java | 2 ++ .../export/generate/BaseGenerator.java | 5 ++++ .../generate/CollectionSourceGenerator.java | 25 ++++++++++++------- .../export/generate/GeneratorPayload.java | 24 ++++++++++++++++-- .../generate/SourceGeneratorFactory.java | 16 +++++++++--- .../generate/TissueSourceGenerator.java | 13 ++++++++++ .../dsm/model/patch/BasePatch.java | 2 +- .../generate/SourceGeneratorFactoryTest.java | 17 +++++++++++++ 8 files changed, 89 insertions(+), 15 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/TissueSourceGenerator.java create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorFactoryTest.java diff --git a/src/main/java/org/broadinstitute/dsm/db/Tissue.java b/src/main/java/org/broadinstitute/dsm/db/Tissue.java index 86abb5aee..6b8b13660 100644 --- a/src/main/java/org/broadinstitute/dsm/db/Tissue.java +++ b/src/main/java/org/broadinstitute/dsm/db/Tissue.java @@ -152,7 +152,9 @@ public Map getDynamicFields() { @ColumnName (DBConstants.H_E_COUNT) private Integer hECount; + public Tissue() { + } public Tissue(String tissueId, String oncHistoryDetailId, String notes, Integer countReceived, String tissueType, String tissueSite, String tumorType, String hE, String pathologyReport, String collaboratorSampleId, diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java index 4f33b8371..be8ad1627 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java @@ -107,6 +107,7 @@ public static class PropertyInfo { private boolean isCollection; private String fieldName; + public PropertyInfo(Class propertyClass, boolean isCollection) { this.propertyClass = Objects.requireNonNull(propertyClass); this.isCollection = isCollection; @@ -139,6 +140,10 @@ public String getFieldName() { if (StringUtils.isBlank(this.fieldName)) this.fieldName = ""; return this.fieldName; } + + public Class getPropertyClass() { + return propertyClass; + } } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java index 9bb63c081..1b78c850b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionSourceGenerator.java @@ -3,6 +3,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import org.broadinstitute.dsm.statics.ESObjectConstants; @@ -25,18 +26,24 @@ public CollectionSourceGenerator() { @Override public Object construct() { logger.info("Constructing nested data"); - Map mapWithParsedObjects = new HashMap<>(Map.of( - getPrimaryKey(), generatorPayload.getRecordId(), - ESObjectConstants.DYNAMIC_FIELDS, parseJsonValuesToObject()) - ); - return List.of(mapWithParsedObjects); + Map fieldNameElement = new HashMap<>(Map.of(ESObjectConstants.DYNAMIC_FIELDS, parseJsonValuesToObject())); + return getCollectionElementMap(fieldNameElement); } @Override protected Object getElement(Object element) { - return List.of(new HashMap<>(Map.of( - getPrimaryKey(), generatorPayload.getRecordId(), - getFieldName(), element) - )); + Map fieldNameElement = new HashMap<>(Map.of(getFieldName(), element)); + return getCollectionElementMap(fieldNameElement); + } + + private List> getCollectionElementMap(Map element) { + HashMap mapWithParsedObjects = new HashMap<>(Map.of(getPrimaryKey(), generatorPayload.getRecordId())); + mapWithParsedObjects.putAll(element); + getParentWithId().ifPresent(mapWithParsedObjects::putAll); + return List.of(mapWithParsedObjects); + } + + protected Optional> getParentWithId() { + return Optional.empty(); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorPayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorPayload.java index 1dbd6fdc9..663612ff7 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorPayload.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorPayload.java @@ -7,12 +7,20 @@ public class GeneratorPayload { NameValue nameValue; int recordId; + private String parent; + private String parentId; public GeneratorPayload(NameValue nameValue, int recordId) { this.nameValue = nameValue; this.recordId = recordId; } + public GeneratorPayload(NameValue nameValue, int recordId, String parent, String parentId) { + this(nameValue, recordId); + this.parent = parent; + this.parentId = parentId; + } + public GeneratorPayload(NameValue nameValue) { this.nameValue = nameValue; } @@ -21,14 +29,26 @@ public NameValue getNameValue() { return nameValue; } - public String getName() {return nameValue.getName();} + public String getName() { + return nameValue.getName(); + } - public Object getValue() { return nameValue.getValue(); } + public Object getValue() { + return nameValue.getValue(); + } public int getRecordId() { return recordId; } + public String getParent() { + return parent; + } + + public String getParentId() { + return parentId; + } + public String getFieldName() { return Util.underscoresToCamelCase(Util.getDBElement(getName()).getColumnName()); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorFactory.java index 17321048e..b8c3bbe12 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorFactory.java @@ -1,11 +1,21 @@ package org.broadinstitute.dsm.model.elastic.export.generate; +import org.broadinstitute.dsm.db.Tissue; + public class SourceGeneratorFactory implements GeneratorFactory { @Override public BaseGenerator make(BaseGenerator.PropertyInfo propertyInfo) { - return propertyInfo.isCollection() - ? new CollectionSourceGenerator() - : new SingleSourceGenerator(); + BaseGenerator generator; + if (propertyInfo.isCollection()) { + if (Tissue.class.isAssignableFrom(propertyInfo.getPropertyClass())) { + generator = new TissueSourceGenerator(); + } else { + generator = new CollectionMappingGenerator(); + } + } else { + generator = new SingleSourceGenerator(); + } + return generator; } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/TissueSourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/TissueSourceGenerator.java new file mode 100644 index 000000000..cd5ec2d1f --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/TissueSourceGenerator.java @@ -0,0 +1,13 @@ +package org.broadinstitute.dsm.model.elastic.export.generate; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +public class TissueSourceGenerator extends CollectionSourceGenerator { + + @Override + protected Optional> getParentWithId() { + return Optional.of(new HashMap<>(Map.of(generatorPayload.getParent(), generatorPayload.getParentId()))); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index 1931d5b9d..59c3b085f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -66,7 +66,7 @@ protected BasePatch(Patch patch) { } private void exportToES(NameValue nameValue) { - GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, Integer.parseInt(patch.getId())); + GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, Integer.parseInt(patch.getId()), patch.getParent(), patch.getParentId()); ExportFacadePayload exportFacadePayload = new ExportFacadePayload(ddpInstance.getParticipantIndexES(), patch.getDdpParticipantId(), generatorPayload); ExportFacade exportFacade = new ExportFacade(exportFacadePayload); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorFactoryTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorFactoryTest.java new file mode 100644 index 000000000..fc50aa5b3 --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorFactoryTest.java @@ -0,0 +1,17 @@ +package org.broadinstitute.dsm.model.elastic.export.generate; + +import org.broadinstitute.dsm.db.Tissue; +import org.junit.Assert; +import org.junit.Test; + +public class SourceGeneratorFactoryTest { + + @Test + public void processMake() { + SourceGeneratorFactory sourceGeneratorFactory = new SourceGeneratorFactory(); + BaseGenerator.PropertyInfo propertyInfo = new BaseGenerator.PropertyInfo(Tissue.class, true); + BaseGenerator generator = sourceGeneratorFactory.make(propertyInfo); + Assert.assertEquals(TissueSourceGenerator.class, generator.getClass()); + } + +} \ No newline at end of file From ce362abeee409ae3291f8d25c1574e688a6e504f Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 14 Dec 2021 10:51:06 +0400 Subject: [PATCH 220/405] DDP-7121 | make corrections in source factory, rename Tissue fields --- src/main/java/org/broadinstitute/dsm/db/Tissue.java | 2 ++ .../elastic/export/generate/CollectionMappingGenerator.java | 3 +-- .../model/elastic/export/generate/SourceGeneratorFactory.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/Tissue.java b/src/main/java/org/broadinstitute/dsm/db/Tissue.java index 44d406ddf..16a4dd68c 100644 --- a/src/main/java/org/broadinstitute/dsm/db/Tissue.java +++ b/src/main/java/org/broadinstitute/dsm/db/Tissue.java @@ -1,5 +1,6 @@ package org.broadinstitute.dsm.db; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.type.TypeReference; @@ -33,6 +34,7 @@ primaryKey = DBConstants.TISSUE_ID, columnPrefix = "") @JsonInclude(JsonInclude.Include.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) public class Tissue { private static final Logger logger = LoggerFactory.getLogger(Tissue.class); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java index 8d7f4e71f..22b7287d3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/CollectionMappingGenerator.java @@ -14,8 +14,7 @@ public CollectionMappingGenerator(Parser parser, GeneratorPayload generatorPaylo public CollectionMappingGenerator() {} @Override - protected Map getElement(Object type) { - return new HashMap<>(Map.of(getFieldName(), type)); + protected Map getElement(Object type) { return new HashMap<>(Map.of(getFieldName(), type)); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorFactory.java index b8c3bbe12..a27417f63 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorFactory.java @@ -11,7 +11,7 @@ public BaseGenerator make(BaseGenerator.PropertyInfo propertyInfo) { if (Tissue.class.isAssignableFrom(propertyInfo.getPropertyClass())) { generator = new TissueSourceGenerator(); } else { - generator = new CollectionMappingGenerator(); + generator = new CollectionSourceGenerator(); } } else { generator = new SingleSourceGenerator(); From 9ff0d07a10f558c3da244ece955a4d28a804dc4e Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 14 Dec 2021 11:21:33 +0400 Subject: [PATCH 221/405] DDP-7121 | minor refactoring --- src/main/java/org/broadinstitute/dsm/db/ViewFilter.java | 8 ++++---- src/main/java/org/broadinstitute/dsm/model/Filter.java | 2 +- .../filter/participant/ManualFilterParticipantList.java | 4 ---- .../org/broadinstitute/dsm/util/ElasticSearchUtil.java | 1 + 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/ViewFilter.java b/src/main/java/org/broadinstitute/dsm/db/ViewFilter.java index 39a34ab3c..a16185f4a 100644 --- a/src/main/java/org/broadinstitute/dsm/db/ViewFilter.java +++ b/src/main/java/org/broadinstitute/dsm/db/ViewFilter.java @@ -253,11 +253,11 @@ private static Result saveFilter(@NonNull ViewFilter viewFilter, @NonNull String public static void addQueryCondition(@NonNull Map queryConditions, DBElement dbElement, Filter filter) { if (dbElement != null) { String queryCondition = ""; - String tmp = StringUtils.isNotBlank(filter.getParentName()) ? filter.getParentName() : filter.getParticipantColumn().getTableAlias(); - if (queryConditions.containsKey(tmp)) { - queryCondition = queryConditions.get(tmp); + String propertyName = StringUtils.isNotBlank(filter.getParentName()) ? filter.getParentName() : filter.getParticipantColumn().getTableAlias(); + if (queryConditions.containsKey(propertyName)) { + queryCondition = queryConditions.get(propertyName); } - queryConditions.put(tmp, queryCondition.concat(Filter.getQueryStringForFiltering(filter, dbElement))); + queryConditions.put(propertyName, queryCondition.concat(Filter.getQueryStringForFiltering(filter, dbElement))); } else { String queryCondition = ""; diff --git a/src/main/java/org/broadinstitute/dsm/model/Filter.java b/src/main/java/org/broadinstitute/dsm/model/Filter.java index 14371b18d..8bb074af4 100644 --- a/src/main/java/org/broadinstitute/dsm/model/Filter.java +++ b/src/main/java/org/broadinstitute/dsm/model/Filter.java @@ -101,9 +101,9 @@ public static String getQueryStringForFiltering(@NonNull Filter filter, DBElemen String finalQuery = ""; String query = ""; String condition = ""; + //simple is better than complex, KISS(Keep It Simple Stupid) if (filter.isEmpty() && !ADDITIONAL_VALUES.equals(filter.getType()) && !JSON_ARRAY.equals(filter.getType())) { finalQuery = AND + filter.getColumnName(dbElement) + IS_NULL + " "; - } else if (filter.isNotEmpty() && !ADDITIONAL_VALUES.equals(filter.getType()) && !JSON_ARRAY.equals(filter.getType())) { finalQuery = AND + filter.getColumnName(dbElement) + IS_NOT_NULL + " "; diff --git a/src/main/java/org/broadinstitute/dsm/model/filter/participant/ManualFilterParticipantList.java b/src/main/java/org/broadinstitute/dsm/model/filter/participant/ManualFilterParticipantList.java index 4b2871e69..3b0ce56fe 100644 --- a/src/main/java/org/broadinstitute/dsm/model/filter/participant/ManualFilterParticipantList.java +++ b/src/main/java/org/broadinstitute/dsm/model/filter/participant/ManualFilterParticipantList.java @@ -1,9 +1,5 @@ package org.broadinstitute.dsm.model.filter.participant; -import java.util.List; - -import org.broadinstitute.dsm.model.participant.ParticipantWrapper; -import org.broadinstitute.dsm.model.participant.ParticipantWrapperDto; import org.broadinstitute.dsm.model.participant.ParticipantWrapperResult; import org.broadinstitute.dsm.util.PatchUtil; import spark.QueryParamsMap; diff --git a/src/main/java/org/broadinstitute/dsm/util/ElasticSearchUtil.java b/src/main/java/org/broadinstitute/dsm/util/ElasticSearchUtil.java index 16afa1597..503df435f 100644 --- a/src/main/java/org/broadinstitute/dsm/util/ElasticSearchUtil.java +++ b/src/main/java/org/broadinstitute/dsm/util/ElasticSearchUtil.java @@ -837,6 +837,7 @@ public static String getPreferredLanguage(@NonNull Map> createESQuery(@NonNull String filter) { String[] filters = filter.split(Filter.AND); BoolQueryBuilder finalQuery = new BoolQueryBuilder(); From ad27b4e67035a5c639bc62a2b760fb0907233add Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 15 Dec 2021 15:37:48 +0400 Subject: [PATCH 222/405] DDP-7121 | introduce DsmAbstarctQueryBuilder abstract class which will serve as a base for creating queries for one-to-one and one-to-many data --- .../filter/CollectionQueryBuilder.java | 19 +++++++ .../filter/DsmAbstractQueryBuilder.java | 14 ++++++ .../filter/DsmAbstractQueryBuilderTest.java | 49 +++++++++++++++++++ 3 files changed, 82 insertions(+) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilderTest.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java new file mode 100644 index 000000000..4c2fc9533 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java @@ -0,0 +1,19 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import org.elasticsearch.index.query.AbstractQueryBuilder; + +public class CollectionQueryBuilder extends DsmAbstractQueryBuilder { + + public CollectionQueryBuilder(String filter) { + super(filter); + } + + @Override + public AbstractQueryBuilder build() { + + String filter = "AND m.medicalRecordId = 15 AND m.dynamicFields.ragac = 55 OR m.medicalRecordName = 213"; + + + return null; + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java new file mode 100644 index 000000000..e670a4497 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java @@ -0,0 +1,14 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import org.elasticsearch.index.query.AbstractQueryBuilder; + +public abstract class DsmAbstractQueryBuilder { + + protected String filter; + + public DsmAbstractQueryBuilder(String filter) { + this.filter = filter; + } + + public abstract AbstractQueryBuilder build(); +} diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilderTest.java new file mode 100644 index 000000000..e2d5c065c --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilderTest.java @@ -0,0 +1,49 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import org.apache.lucene.search.join.ScoreMode; +import org.elasticsearch.index.query.AbstractQueryBuilder; +import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.index.query.NestedQueryBuilder; +import org.junit.Assert; +import org.junit.Test; + +public class DsmAbstractQueryBuilderTest { + + @Test + public void collectionBuild() { + + String filter = "AND m.medicalRecordId = 15"; + + DsmAbstractQueryBuilder dsmAbstractQueryBuilder = new CollectionQueryBuilder(filter); + + AbstractQueryBuilder query = dsmAbstractQueryBuilder.build(); + + String filterJson = query.toString(); + String toBeContained = "\"dsm.medicalRecord.medicalRecordId\": \"15\""; + Assert.assertTrue(filterJson.contains(toBeContained)); + } + + @Test + public void singleBuild() { + + String filter = "AND m.medicalRecordId = 15"; + + DsmAbstractQueryBuilder dsmAbstractQueryBuilder = new DsmAbstractQueryBuilder(filter) { + + @Override + public AbstractQueryBuilder build() { + + return null; + } + }; + + String expected = ""; + + AbstractQueryBuilder query = dsmAbstractQueryBuilder.build(); + + Assert.assertEquals(expected, query); + + } + + +} From 46b00febc1530f3a06fe8c1a7c1a2dd169f95279 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 15 Dec 2021 16:27:41 +0400 Subject: [PATCH 223/405] DDP-7121 | start making parser --- .../filter/CollectionQueryBuilder.java | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java index 4c2fc9533..844831fde 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java @@ -1,5 +1,11 @@ package org.broadinstitute.dsm.model.elastic.filter; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + import org.elasticsearch.index.query.AbstractQueryBuilder; public class CollectionQueryBuilder extends DsmAbstractQueryBuilder { @@ -10,9 +16,27 @@ public CollectionQueryBuilder(String filter) { @Override public AbstractQueryBuilder build() { - + Map> filterByLogicalOperators = new HashMap<>(); String filter = "AND m.medicalRecordId = 15 AND m.dynamicFields.ragac = 55 OR m.medicalRecordName = 213"; + String[] andSeparated = filter.split("AND"); + for (String eachFilter : andSeparated) { + String cleanedEachFilter = eachFilter.trim(); + if (cleanedEachFilter.contains("OR")) { + + } else { + filterByLogicalOperators.compute("AND", (curr, prev) -> { + if (Objects.isNull(prev)) { + List filters = new ArrayList<>(); + filters.add(cleanedEachFilter); + return filters; + } else { + prev.add(cleanedEachFilter); + return prev; + } + }); + } + } return null; } From f808209f0bafd18da4f034c3ad6087ee71ec5e1a Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 15 Dec 2021 17:40:32 +0400 Subject: [PATCH 224/405] DDP-7121 | introduce parseFiltersByLogicalOperators() in CollectionQueryBuilder.java --- .../dsm/model/elastic/Util.java | 6 ++ .../export/generate/BaseGenerator.java | 4 +- .../filter/CollectionQueryBuilder.java | 62 ++++++++++++------- .../filter/DsmAbstractQueryBuilder.java | 3 + .../filter/CollectionQueryBuilderTest.java | 28 +++++++++ 5 files changed, 77 insertions(+), 26 deletions(-) create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index b7b96e06e..e47c0f2dd 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -214,6 +214,12 @@ public static String camelCaseToPascalSnakeCase(String camelCase) { return pascalSnakeCase; } + public static String capitalCamelCaseToLowerCamelCase(String capitalCamelCase) { + StringBuilder className = new StringBuilder(capitalCamelCase); + StringBuilder camelCaseClassName = className.replace(0, 1, String.valueOf(className.charAt(0)).toLowerCase()); + return camelCaseClassName.toString(); + } + public static class Constants { public static final String PROFILE = "profile"; public static final String PROFILE_HRUID = PROFILE + ".hruid"; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java index be8ad1627..69be19655 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java @@ -118,9 +118,7 @@ public void setIsCollection(boolean isCollection) { } public String getPropertyName() { - StringBuilder className = new StringBuilder(propertyClass.getSimpleName()); - StringBuilder camelCaseClassName = className.replace(0, 1, String.valueOf(className.charAt(0)).toLowerCase()); - return camelCaseClassName.toString(); + return Util.capitalCamelCaseToLowerCamelCase(propertyClass.getSimpleName()); } public String getPrimaryKey() { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java index 844831fde..9a1080d1e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java @@ -1,12 +1,13 @@ package org.broadinstitute.dsm.model.elastic.filter; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; -import org.elasticsearch.index.query.AbstractQueryBuilder; +import org.apache.commons.lang3.StringUtils; +import org.apache.lucene.search.join.ScoreMode; +import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.statics.ESObjectConstants; +import org.broadinstitute.dsm.util.ElasticSearchUtil; +import org.elasticsearch.index.query.*; public class CollectionQueryBuilder extends DsmAbstractQueryBuilder { @@ -16,28 +17,43 @@ public CollectionQueryBuilder(String filter) { @Override public AbstractQueryBuilder build() { - Map> filterByLogicalOperators = new HashMap<>(); - String filter = "AND m.medicalRecordId = 15 AND m.dynamicFields.ragac = 55 OR m.medicalRecordName = 213"; + Map> parsedFilters = parseFiltersByLogicalOperators(); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + for (Map.Entry> parsedFilter: parsedFilters.entrySet()) { + if (parsedFilter.getKey().equals("AND")) { + List filterValues = parsedFilter.getValue(); + for (String filterValue : filterValues) { + String[] splittedFilter = filterValue.split("="); + String value = splittedFilter[1].trim(); + String[] aliasWithField = splittedFilter[0].trim().split(ElasticSearchUtil.DOT_SEPARATOR); + String innerProperty = aliasWithField[1];// medicalRecordId + String alias = aliasWithField[0]; + String outerProperty = Util.TABLE_ALIAS_MAPPINGS.get(alias).getPropertyName(); //medicalRecord + String nestedPath = DSM_WITH_DOT + outerProperty; + boolQueryBuilder.must(new NestedQueryBuilder(nestedPath, new MatchQueryBuilder(nestedPath + "." + innerProperty, value), ScoreMode.Avg)); + } + } else { + boolQueryBuilder.should(); + } + } + return null; + } + + protected Map> parseFiltersByLogicalOperators() { + Map> filterByLogicalOperators = new HashMap<>(Map.of("AND", new ArrayList<>(), "OR", new ArrayList<>())); String[] andSeparated = filter.split("AND"); for (String eachFilter : andSeparated) { String cleanedEachFilter = eachFilter.trim(); if (cleanedEachFilter.contains("OR")) { - - } else { - filterByLogicalOperators.compute("AND", (curr, prev) -> { - if (Objects.isNull(prev)) { - List filters = new ArrayList<>(); - filters.add(cleanedEachFilter); - return filters; - } else { - prev.add(cleanedEachFilter); - return prev; - } - }); - + String[] orSeparated = cleanedEachFilter.split("OR"); + filterByLogicalOperators.get("AND").add(orSeparated[0].trim()); + Arrays.stream(orSeparated) + .skip(1) + .forEach(f -> filterByLogicalOperators.get("OR").add(f.trim())); + } else if (StringUtils.isNotBlank(cleanedEachFilter)){ + filterByLogicalOperators.get("AND").add(cleanedEachFilter); } } - - return null; + return filterByLogicalOperators; } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java index e670a4497..f738bfdeb 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java @@ -1,9 +1,12 @@ package org.broadinstitute.dsm.model.elastic.filter; +import org.broadinstitute.dsm.statics.DBConstants; +import org.broadinstitute.dsm.statics.ESObjectConstants; import org.elasticsearch.index.query.AbstractQueryBuilder; public abstract class DsmAbstractQueryBuilder { + protected static final String DSM_WITH_DOT = ESObjectConstants.DSM + DBConstants.ALIAS_DELIMITER; protected String filter; public DsmAbstractQueryBuilder(String filter) { diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java new file mode 100644 index 000000000..3813e60ea --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java @@ -0,0 +1,28 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.*; + +public class CollectionQueryBuilderTest { + + @Test + public void parseFiltersByLogicalOperators() { + String filter = "AND m.medicalRecordId = 15 OR m.medicalRecordSomething LIKE 55555 OR m.medicalRecordSomethingg = 55552 AND m.dynamicFields.ragac = 55 OR m.medicalRecordName = 213"; + CollectionQueryBuilder collectionQueryBuilder = new CollectionQueryBuilder(filter); + Map> parsedFilters = collectionQueryBuilder.parseFiltersByLogicalOperators(); + for (Map.Entry> eachFilter: parsedFilters.entrySet()) { + if (eachFilter.getKey().equals("AND")) { + Assert.assertArrayEquals(new ArrayList<>(List.of("m.medicalRecordId = 15", "m.dynamicFields.ragac = 55")).toArray(), eachFilter.getValue().toArray()); + } else { + Assert.assertArrayEquals(new ArrayList<>(List.of("m.medicalRecordSomething LIKE 55555", "m.medicalRecordSomethingg = 55552", "m.medicalRecordName = 213")).toArray(), eachFilter.getValue().toArray()); + } + } + } +} \ No newline at end of file From 71e7933024329bc53c717956eb95ee1849ba95f5 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 15 Dec 2021 18:15:14 +0400 Subject: [PATCH 225/405] DDP-7121 | add filter strategy, refactor test --- .../filter/CollectionQueryBuilder.java | 35 ++++++++++++------- .../model/elastic/filter/FilterStrategy.java | 10 ++++++ .../filter/DsmAbstractQueryBuilderTest.java | 6 ++-- 3 files changed, 36 insertions(+), 15 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/FilterStrategy.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java index 9a1080d1e..4be9eb0e0 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java @@ -20,23 +20,31 @@ public AbstractQueryBuilder build() { Map> parsedFilters = parseFiltersByLogicalOperators(); BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); for (Map.Entry> parsedFilter: parsedFilters.entrySet()) { + List filterValues = parsedFilter.getValue(); if (parsedFilter.getKey().equals("AND")) { - List filterValues = parsedFilter.getValue(); - for (String filterValue : filterValues) { - String[] splittedFilter = filterValue.split("="); - String value = splittedFilter[1].trim(); - String[] aliasWithField = splittedFilter[0].trim().split(ElasticSearchUtil.DOT_SEPARATOR); - String innerProperty = aliasWithField[1];// medicalRecordId - String alias = aliasWithField[0]; - String outerProperty = Util.TABLE_ALIAS_MAPPINGS.get(alias).getPropertyName(); //medicalRecord - String nestedPath = DSM_WITH_DOT + outerProperty; - boolQueryBuilder.must(new NestedQueryBuilder(nestedPath, new MatchQueryBuilder(nestedPath + "." + innerProperty, value), ScoreMode.Avg)); - } + buildUpNestedQuery(boolQueryBuilder, filterValues, BoolQueryBuilder::must); } else { - boolQueryBuilder.should(); + buildUpNestedQuery(boolQueryBuilder, filterValues, BoolQueryBuilder::should); } } - return null; + return boolQueryBuilder; + } + + private void buildUpNestedQuery(BoolQueryBuilder boolQueryBuilder, List filterValues, FilterStrategy filterStrategy) { + for (String filterValue : filterValues) { + String[] splittedFilter = filterValue.split("="); + String value = splittedFilter[1].trim(); + String[] aliasWithField = splittedFilter[0].trim().split(ElasticSearchUtil.DOT_SEPARATOR); + String innerProperty = aliasWithField[1];// medicalRecordId + String alias = aliasWithField[0]; + String outerProperty = Util.TABLE_ALIAS_MAPPINGS.get(alias).getPropertyName(); //medicalRecord + String nestedPath = DSM_WITH_DOT + outerProperty; + filterStrategy.build(boolQueryBuilder, buildNestedQueryBuilder(nestedPath, innerProperty, value)); + } + } + + private NestedQueryBuilder buildNestedQueryBuilder(String path, String fieldName, Object value) { + return new NestedQueryBuilder(path, new MatchQueryBuilder(path + "." + fieldName, value), ScoreMode.Avg); } protected Map> parseFiltersByLogicalOperators() { @@ -57,3 +65,4 @@ protected Map> parseFiltersByLogicalOperators() { return filterByLogicalOperators; } } + diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/FilterStrategy.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/FilterStrategy.java new file mode 100644 index 000000000..e22a3a668 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/FilterStrategy.java @@ -0,0 +1,10 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; + +public interface FilterStrategy { + + void build(BoolQueryBuilder boolQueryBuilder, QueryBuilder queryBuilder); + +} diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilderTest.java index e2d5c065c..0338a93a6 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilderTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilderTest.java @@ -19,8 +19,10 @@ public void collectionBuild() { AbstractQueryBuilder query = dsmAbstractQueryBuilder.build(); String filterJson = query.toString(); - String toBeContained = "\"dsm.medicalRecord.medicalRecordId\": \"15\""; - Assert.assertTrue(filterJson.contains(toBeContained)); + String nestedKey = "\"dsm.medicalRecord.medicalRecordId\" : "; + String queryValue = "\"query\" : \"15\""; + Assert.assertTrue(filterJson.contains(nestedKey)); + Assert.assertTrue(filterJson.contains(queryValue)); } @Test From 6f9a4a300d19ddb28e3640ac6bf23ecfccc87ed4 Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 16 Dec 2021 10:11:06 +0400 Subject: [PATCH 226/405] DDP-7121 | extract static methods in Util.java, add OR case in CollectionQueryBuilde.java, make tests pass --- .../dsm/model/elastic/Util.java | 10 ++---- .../filter/CollectionQueryBuilder.java | 21 ++++++------ .../filter/DsmAbstractQueryBuilderTest.java | 32 +++++-------------- 3 files changed, 22 insertions(+), 41 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index e47c0f2dd..d9d29cb00 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -13,12 +13,7 @@ import com.google.common.reflect.TypeToken; import com.google.gson.Gson; import org.apache.commons.lang3.StringUtils; -import org.broadinstitute.dsm.db.MedicalRecord; -import org.broadinstitute.dsm.db.OncHistory; -import org.broadinstitute.dsm.db.OncHistoryDetail; -import org.broadinstitute.dsm.db.Participant; -import org.broadinstitute.dsm.db.ParticipantData; -import org.broadinstitute.dsm.db.Tissue; +import org.broadinstitute.dsm.db.*; import org.broadinstitute.dsm.db.structure.ColumnName; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; @@ -38,7 +33,8 @@ public class Util { DBConstants.DDP_PARTICIPANT_DATA_ALIAS, new BaseGenerator.PropertyInfo(ParticipantData.class, true), DBConstants.DDP_PARTICIPANT_RECORD_ALIAS, new BaseGenerator.PropertyInfo(Participant.class, false), DBConstants.DDP_PARTICIPANT_ALIAS, new BaseGenerator.PropertyInfo(Participant.class, false), - DBConstants.DDP_ONC_HISTORY_ALIAS, new BaseGenerator.PropertyInfo(OncHistory.class, false) + DBConstants.DDP_ONC_HISTORY_ALIAS, new BaseGenerator.PropertyInfo(OncHistory.class, false), + DBConstants.DDP_KIT_REQUEST_ALIAS, new BaseGenerator.PropertyInfo(KitRequestShipping.class, true) ); public static final int FIRST_ELEMENT_INDEX = 0; public static final String UNDERSCORE_SEPARATOR = "_"; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java index 4be9eb0e0..3b35d130c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java @@ -4,8 +4,8 @@ import org.apache.commons.lang3.StringUtils; import org.apache.lucene.search.join.ScoreMode; +import org.broadinstitute.dsm.model.Filter; import org.broadinstitute.dsm.model.elastic.Util; -import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.elasticsearch.index.query.*; @@ -21,7 +21,7 @@ public AbstractQueryBuilder build() { BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); for (Map.Entry> parsedFilter: parsedFilters.entrySet()) { List filterValues = parsedFilter.getValue(); - if (parsedFilter.getKey().equals("AND")) { + if (parsedFilter.getKey().equals(Filter.AND)) { buildUpNestedQuery(boolQueryBuilder, filterValues, BoolQueryBuilder::must); } else { buildUpNestedQuery(boolQueryBuilder, filterValues, BoolQueryBuilder::should); @@ -32,7 +32,8 @@ public AbstractQueryBuilder build() { private void buildUpNestedQuery(BoolQueryBuilder boolQueryBuilder, List filterValues, FilterStrategy filterStrategy) { for (String filterValue : filterValues) { - String[] splittedFilter = filterValue.split("="); + String splitter = filterValue.contains(Filter.EQUALS) ? Filter.EQUALS : Filter.LIKE; + String[] splittedFilter = filterValue.split(splitter); String value = splittedFilter[1].trim(); String[] aliasWithField = splittedFilter[0].trim().split(ElasticSearchUtil.DOT_SEPARATOR); String innerProperty = aliasWithField[1];// medicalRecordId @@ -48,18 +49,18 @@ private NestedQueryBuilder buildNestedQueryBuilder(String path, String fieldName } protected Map> parseFiltersByLogicalOperators() { - Map> filterByLogicalOperators = new HashMap<>(Map.of("AND", new ArrayList<>(), "OR", new ArrayList<>())); - String[] andSeparated = filter.split("AND"); + Map> filterByLogicalOperators = new HashMap<>(Map.of(Filter.AND, new ArrayList<>(), Filter.OR, new ArrayList<>())); + String[] andSeparated = filter.split(Filter.AND); for (String eachFilter : andSeparated) { String cleanedEachFilter = eachFilter.trim(); - if (cleanedEachFilter.contains("OR")) { - String[] orSeparated = cleanedEachFilter.split("OR"); - filterByLogicalOperators.get("AND").add(orSeparated[0].trim()); + if (cleanedEachFilter.contains(Filter.OR)) { + String[] orSeparated = cleanedEachFilter.split(Filter.OR); + filterByLogicalOperators.get(Filter.AND).add(orSeparated[0].trim()); Arrays.stream(orSeparated) .skip(1) - .forEach(f -> filterByLogicalOperators.get("OR").add(f.trim())); + .forEach(f -> filterByLogicalOperators.get(Filter.OR).add(f.trim())); } else if (StringUtils.isNotBlank(cleanedEachFilter)){ - filterByLogicalOperators.get("AND").add(cleanedEachFilter); + filterByLogicalOperators.get(Filter.AND).add(cleanedEachFilter); } } return filterByLogicalOperators; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilderTest.java index 0338a93a6..46b370b1a 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilderTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilderTest.java @@ -2,6 +2,7 @@ import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.index.query.AbstractQueryBuilder; +import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.NestedQueryBuilder; import org.junit.Assert; @@ -12,40 +13,23 @@ public class DsmAbstractQueryBuilderTest { @Test public void collectionBuild() { - String filter = "AND m.medicalRecordId = 15"; + String filter = "AND m.medicalRecordId = 15 AND m.type = PHYSICIAN OR k.bspCollaboratorSampleId = ASCProject_PZ8GJC_SALIVA"; DsmAbstractQueryBuilder dsmAbstractQueryBuilder = new CollectionQueryBuilder(filter); - AbstractQueryBuilder query = dsmAbstractQueryBuilder.build(); + AbstractQueryBuilder actual = dsmAbstractQueryBuilder.build(); - String filterJson = query.toString(); - String nestedKey = "\"dsm.medicalRecord.medicalRecordId\" : "; - String queryValue = "\"query\" : \"15\""; - Assert.assertTrue(filterJson.contains(nestedKey)); - Assert.assertTrue(filterJson.contains(queryValue)); + AbstractQueryBuilder expected = new BoolQueryBuilder().must(new NestedQueryBuilder("dsm.medicalRecord", new MatchQueryBuilder("dsm.medicalRecord.medicalRecordId", "15"), ScoreMode.Avg)) + .must(new NestedQueryBuilder("dsm.medicalRecord", new MatchQueryBuilder("dsm.medicalRecord.type", "PHYSICIAN"), ScoreMode.Avg)) + .should(new NestedQueryBuilder("dsm.kitRequestShipping", new MatchQueryBuilder("dsm.kitRequestShipping.bspCollaboratorSampleId", "ASCProject_PZ8GJC_SALIVA"), ScoreMode.Avg)); + + Assert.assertEquals(expected, actual); } @Test public void singleBuild() { - String filter = "AND m.medicalRecordId = 15"; - - DsmAbstractQueryBuilder dsmAbstractQueryBuilder = new DsmAbstractQueryBuilder(filter) { - - @Override - public AbstractQueryBuilder build() { - - return null; - } - }; - - String expected = ""; - - AbstractQueryBuilder query = dsmAbstractQueryBuilder.build(); - - Assert.assertEquals(expected, query); } - } From f36901c8b828cde37efbc6e67bfd1c8b2912ee05 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 16 Dec 2021 12:23:53 +0400 Subject: [PATCH 227/405] DDP-7121 | attempt to apply visitor pattern for splitting by different criterias --- .../filter/CollectionQueryBuilder.java | 14 +++++--- .../model/elastic/filter/EqualsVisitor.java | 15 ++++++++ .../filter/GreaterThanOrEqualVisitor.java | 13 +++++++ .../filter/LessThanOrEqualVisitor.java | 14 ++++++++ .../dsm/model/elastic/filter/LikeVisitor.java | 13 +++++++ .../elastic/filter/SpliteratorVisitor.java | 21 +++++++++++ .../dsm/model/elastic/filter/Unit.java | 36 +++++++++++++++++++ .../model/elastic/filter/VisitorFactory.java | 4 +++ 8 files changed, 125 insertions(+), 5 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/EqualsVisitor.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/GreaterThanOrEqualVisitor.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/LessThanOrEqualVisitor.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/LikeVisitor.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/SpliteratorVisitor.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/Unit.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/VisitorFactory.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java index 3b35d130c..7f8f478b0 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java @@ -32,10 +32,14 @@ public AbstractQueryBuilder build() { private void buildUpNestedQuery(BoolQueryBuilder boolQueryBuilder, List filterValues, FilterStrategy filterStrategy) { for (String filterValue : filterValues) { - String splitter = filterValue.contains(Filter.EQUALS) ? Filter.EQUALS : Filter.LIKE; - String[] splittedFilter = filterValue.split(splitter); - String value = splittedFilter[1].trim(); - String[] aliasWithField = splittedFilter[0].trim().split(ElasticSearchUtil.DOT_SEPARATOR); + SpliteratorVisitor visitor = SpliteratorVisitor.instance(filterValue); + Unit unit = new Unit(filterValue); + unit.accept(visitor); + List splittedFilter = unit.getSplittedWords(); +// String splitter = filterValue.contains(Filter.EQUALS) ? Filter.EQUALS : Filter.LIKE; +// String[] splittedFilter = filterValue.split(splitter); + String value = splittedFilter.get(1).trim(); + String[] aliasWithField = splittedFilter.get(0).trim().split(ElasticSearchUtil.DOT_SEPARATOR); String innerProperty = aliasWithField[1];// medicalRecordId String alias = aliasWithField[0]; String outerProperty = Util.TABLE_ALIAS_MAPPINGS.get(alias).getPropertyName(); //medicalRecord @@ -50,7 +54,7 @@ private NestedQueryBuilder buildNestedQueryBuilder(String path, String fieldName protected Map> parseFiltersByLogicalOperators() { Map> filterByLogicalOperators = new HashMap<>(Map.of(Filter.AND, new ArrayList<>(), Filter.OR, new ArrayList<>())); - String[] andSeparated = filter.split(Filter.AND); + String[] andSeparated = filter.split(Filter.AND_TRIMMED); for (String eachFilter : andSeparated) { String cleanedEachFilter = eachFilter.trim(); if (cleanedEachFilter.contains(Filter.OR)) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/EqualsVisitor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/EqualsVisitor.java new file mode 100644 index 000000000..875910db8 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/EqualsVisitor.java @@ -0,0 +1,15 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import java.util.Arrays; + +import org.broadinstitute.dsm.model.Filter; + +public class EqualsVisitor implements SpliteratorVisitor { + + @Override + public void visit(Unit equalsLikeUnit) { + equalsLikeUnit.splittedWords = Arrays.asList(equalsLikeUnit.valueToSplit.split(Filter.EQUALS_TRIMMED)); + } + + +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/GreaterThanOrEqualVisitor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/GreaterThanOrEqualVisitor.java new file mode 100644 index 000000000..6cf3ee515 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/GreaterThanOrEqualVisitor.java @@ -0,0 +1,13 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import java.util.Arrays; + +import org.broadinstitute.dsm.model.Filter; + +public class GreaterThanOrEqualVisitor implements SpliteratorVisitor { + + @Override + public void visit(Unit greaterThanOrEqualUnit) { + greaterThanOrEqualUnit.splittedWords = Arrays.asList(greaterThanOrEqualUnit.valueToSplit.split(Filter.LARGER_EQUALS_TRIMMED)); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LessThanOrEqualVisitor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LessThanOrEqualVisitor.java new file mode 100644 index 000000000..db1496e89 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LessThanOrEqualVisitor.java @@ -0,0 +1,14 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import java.util.Arrays; + +import org.broadinstitute.dsm.model.Filter; + +public class LessThanOrEqualVisitor implements SpliteratorVisitor { + + + @Override + public void visit(Unit lessThanOrEqualUnit) { + lessThanOrEqualUnit.splittedWords = Arrays.asList(lessThanOrEqualUnit.valueToSplit.split(Filter.SMALLER_EQUALS_TRIMMED)); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LikeVisitor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LikeVisitor.java new file mode 100644 index 000000000..a15ac8290 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LikeVisitor.java @@ -0,0 +1,13 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import java.util.Arrays; + +import org.broadinstitute.dsm.model.Filter; + +public class LikeVisitor implements SpliteratorVisitor{ + + @Override + public void visit(Unit unit) { + unit.splittedWords = Arrays.asList(unit.valueToSplit.split(Filter.LIKE_TRIMMED)); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SpliteratorVisitor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SpliteratorVisitor.java new file mode 100644 index 000000000..715417ab2 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SpliteratorVisitor.java @@ -0,0 +1,21 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import org.broadinstitute.dsm.model.Filter; + +public interface SpliteratorVisitor { + + void visit(Unit unit); + + static SpliteratorVisitor instance(String filterValues) { + SpliteratorVisitor spliteratorVisitor; + if (filterValues.contains(Filter.EQUALS_TRIMMED)) { + spliteratorVisitor = new EqualsVisitor(); + } else if (filterValues.contains(Filter.LIKE_TRIMMED)) { + spliteratorVisitor = new LikeVisitor(); + } else { + throw new IllegalArgumentException("Unknown operator"); + } + return spliteratorVisitor; + } + +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Unit.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Unit.java new file mode 100644 index 000000000..532a3b5ec --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Unit.java @@ -0,0 +1,36 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import java.util.List; +import java.util.Objects; + +public class Unit { + + protected List splittedWords; + protected String valueToSplit; + + public List getSplittedWords() { + return splittedWords; + } + + public void setSplittedWords(List splittedWords) { + this.splittedWords = splittedWords; + } + + public String getValueToSplit() { + return valueToSplit; + } + + public void setValueToSplit(String valueToSplit) { + this.valueToSplit = valueToSplit; + } + + + public Unit(String valueToSplit) { + this.valueToSplit = Objects.requireNonNull(valueToSplit); + } + + public void accept(SpliteratorVisitor visitor) { + visitor.visit(this); + } + +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/VisitorFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/VisitorFactory.java new file mode 100644 index 000000000..54652dd07 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/VisitorFactory.java @@ -0,0 +1,4 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +public class VisitorFactory { +} From dcd0c9a6efee459693facdb3278444a1c778ccd1 Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 16 Dec 2021 15:06:47 +0400 Subject: [PATCH 228/405] DDP-7121 | create SplitterFactory for different kind of splitters based on logical operators --- .../model/elastic/filter/BaseSplitter.java | 32 +++++++++++++++++ .../filter/CollectionQueryBuilder.java | 15 ++------ .../model/elastic/filter/EqualsSplitter.java | 16 +++++++++ .../model/elastic/filter/EqualsVisitor.java | 15 -------- .../filter/GreaterThanEqualsSplitter.java | 15 ++++++++ .../filter/GreaterThanOrEqualVisitor.java | 13 ------- .../elastic/filter/IsNotNullSplitter.java | 15 ++++++++ .../filter/LessThanEqualsSplitter.java | 15 ++++++++ .../filter/LessThanOrEqualVisitor.java | 14 -------- .../model/elastic/filter/LikeSplitter.java | 17 +++++++++ .../dsm/model/elastic/filter/LikeVisitor.java | 13 ------- .../elastic/filter/SpliteratorVisitor.java | 21 ----------- .../model/elastic/filter/SplitterFactory.java | 23 ++++++++++++ .../dsm/model/elastic/filter/Unit.java | 36 ------------------- .../model/elastic/filter/VisitorFactory.java | 4 --- .../dsm/model/elastic/UtilTest.java | 1 + 16 files changed, 137 insertions(+), 128 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitter.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/EqualsSplitter.java delete mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/EqualsVisitor.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/GreaterThanEqualsSplitter.java delete mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/GreaterThanOrEqualVisitor.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/IsNotNullSplitter.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/LessThanEqualsSplitter.java delete mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/LessThanOrEqualVisitor.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/LikeSplitter.java delete mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/LikeVisitor.java delete mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/SpliteratorVisitor.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java delete mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/Unit.java delete mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/VisitorFactory.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitter.java new file mode 100644 index 000000000..1064510c4 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitter.java @@ -0,0 +1,32 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import org.broadinstitute.dsm.util.ElasticSearchUtil; + +public abstract class BaseSplitter { + + protected String filter; + protected String[] splittedFilter; + + public BaseSplitter(String filter) { + this.filter = filter; + splittedFilter = split(); + } + + public abstract String[] split(); + + public String getValue() { + return splittedFilter[1].trim(); + } + + public String getAlias() { + return getFieldWithAlias()[0]; + } + + public String getInnerProperty() { + return getFieldWithAlias()[1]; + } + + private String[] getFieldWithAlias() { + return splittedFilter[0].trim().split(ElasticSearchUtil.DOT_SEPARATOR); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java index 7f8f478b0..b2890b18a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java @@ -32,19 +32,10 @@ public AbstractQueryBuilder build() { private void buildUpNestedQuery(BoolQueryBuilder boolQueryBuilder, List filterValues, FilterStrategy filterStrategy) { for (String filterValue : filterValues) { - SpliteratorVisitor visitor = SpliteratorVisitor.instance(filterValue); - Unit unit = new Unit(filterValue); - unit.accept(visitor); - List splittedFilter = unit.getSplittedWords(); -// String splitter = filterValue.contains(Filter.EQUALS) ? Filter.EQUALS : Filter.LIKE; -// String[] splittedFilter = filterValue.split(splitter); - String value = splittedFilter.get(1).trim(); - String[] aliasWithField = splittedFilter.get(0).trim().split(ElasticSearchUtil.DOT_SEPARATOR); - String innerProperty = aliasWithField[1];// medicalRecordId - String alias = aliasWithField[0]; - String outerProperty = Util.TABLE_ALIAS_MAPPINGS.get(alias).getPropertyName(); //medicalRecord + BaseSplitter splitter = SplitterFactory.createSplitter(filterValue); + String outerProperty = Util.TABLE_ALIAS_MAPPINGS.get(splitter.getAlias()).getPropertyName(); //medicalRecord String nestedPath = DSM_WITH_DOT + outerProperty; - filterStrategy.build(boolQueryBuilder, buildNestedQueryBuilder(nestedPath, innerProperty, value)); + filterStrategy.build(boolQueryBuilder, buildNestedQueryBuilder(nestedPath, splitter.getInnerProperty(), splitter.getValue())); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/EqualsSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/EqualsSplitter.java new file mode 100644 index 000000000..dd8eaf239 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/EqualsSplitter.java @@ -0,0 +1,16 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import org.broadinstitute.dsm.model.Filter; + +public class EqualsSplitter extends BaseSplitter { + + + public EqualsSplitter(String filter) { + super(filter); + } + + @Override + public String[] split() { + return filter.split(Filter.EQUALS_TRIMMED); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/EqualsVisitor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/EqualsVisitor.java deleted file mode 100644 index 875910db8..000000000 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/EqualsVisitor.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.broadinstitute.dsm.model.elastic.filter; - -import java.util.Arrays; - -import org.broadinstitute.dsm.model.Filter; - -public class EqualsVisitor implements SpliteratorVisitor { - - @Override - public void visit(Unit equalsLikeUnit) { - equalsLikeUnit.splittedWords = Arrays.asList(equalsLikeUnit.valueToSplit.split(Filter.EQUALS_TRIMMED)); - } - - -} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/GreaterThanEqualsSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/GreaterThanEqualsSplitter.java new file mode 100644 index 000000000..98a859df8 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/GreaterThanEqualsSplitter.java @@ -0,0 +1,15 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import org.broadinstitute.dsm.model.Filter; + +public class GreaterThanEqualsSplitter extends BaseSplitter { + + public GreaterThanEqualsSplitter(String filter) { + super(filter); + } + + @Override + public String[] split() { + return filter.split(Filter.LARGER_EQUALS_TRIMMED); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/GreaterThanOrEqualVisitor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/GreaterThanOrEqualVisitor.java deleted file mode 100644 index 6cf3ee515..000000000 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/GreaterThanOrEqualVisitor.java +++ /dev/null @@ -1,13 +0,0 @@ -package org.broadinstitute.dsm.model.elastic.filter; - -import java.util.Arrays; - -import org.broadinstitute.dsm.model.Filter; - -public class GreaterThanOrEqualVisitor implements SpliteratorVisitor { - - @Override - public void visit(Unit greaterThanOrEqualUnit) { - greaterThanOrEqualUnit.splittedWords = Arrays.asList(greaterThanOrEqualUnit.valueToSplit.split(Filter.LARGER_EQUALS_TRIMMED)); - } -} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/IsNotNullSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/IsNotNullSplitter.java new file mode 100644 index 000000000..b61dbedad --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/IsNotNullSplitter.java @@ -0,0 +1,15 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import org.broadinstitute.dsm.model.Filter; + +public class IsNotNullSplitter extends BaseSplitter { + + public IsNotNullSplitter(String filter) { + super(filter); + } + + @Override + public String[] split() { + return filter.split(Filter.IS_NOT_NULL.trim()); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LessThanEqualsSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LessThanEqualsSplitter.java new file mode 100644 index 000000000..487c5b100 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LessThanEqualsSplitter.java @@ -0,0 +1,15 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import org.broadinstitute.dsm.model.Filter; + +public class LessThanEqualsSplitter extends BaseSplitter { + + public LessThanEqualsSplitter(String filter) { + super(filter); + } + + @Override + public String[] split() { + return filter.split(Filter.SMALLER_EQUALS_TRIMMED); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LessThanOrEqualVisitor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LessThanOrEqualVisitor.java deleted file mode 100644 index db1496e89..000000000 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LessThanOrEqualVisitor.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.broadinstitute.dsm.model.elastic.filter; - -import java.util.Arrays; - -import org.broadinstitute.dsm.model.Filter; - -public class LessThanOrEqualVisitor implements SpliteratorVisitor { - - - @Override - public void visit(Unit lessThanOrEqualUnit) { - lessThanOrEqualUnit.splittedWords = Arrays.asList(lessThanOrEqualUnit.valueToSplit.split(Filter.SMALLER_EQUALS_TRIMMED)); - } -} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LikeSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LikeSplitter.java new file mode 100644 index 000000000..c338f2663 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LikeSplitter.java @@ -0,0 +1,17 @@ +package org.broadinstitute.dsm.model.elastic.filter; + + +import org.broadinstitute.dsm.model.Filter; + +public class LikeSplitter extends BaseSplitter { + + + public LikeSplitter(String filter) { + super(filter); + } + + @Override + public String[] split() { + return filter.split(Filter.EQUALS_TRIMMED); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LikeVisitor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LikeVisitor.java deleted file mode 100644 index a15ac8290..000000000 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LikeVisitor.java +++ /dev/null @@ -1,13 +0,0 @@ -package org.broadinstitute.dsm.model.elastic.filter; - -import java.util.Arrays; - -import org.broadinstitute.dsm.model.Filter; - -public class LikeVisitor implements SpliteratorVisitor{ - - @Override - public void visit(Unit unit) { - unit.splittedWords = Arrays.asList(unit.valueToSplit.split(Filter.LIKE_TRIMMED)); - } -} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SpliteratorVisitor.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SpliteratorVisitor.java deleted file mode 100644 index 715417ab2..000000000 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SpliteratorVisitor.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.broadinstitute.dsm.model.elastic.filter; - -import org.broadinstitute.dsm.model.Filter; - -public interface SpliteratorVisitor { - - void visit(Unit unit); - - static SpliteratorVisitor instance(String filterValues) { - SpliteratorVisitor spliteratorVisitor; - if (filterValues.contains(Filter.EQUALS_TRIMMED)) { - spliteratorVisitor = new EqualsVisitor(); - } else if (filterValues.contains(Filter.LIKE_TRIMMED)) { - spliteratorVisitor = new LikeVisitor(); - } else { - throw new IllegalArgumentException("Unknown operator"); - } - return spliteratorVisitor; - } - -} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java new file mode 100644 index 000000000..441a96051 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java @@ -0,0 +1,23 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import org.broadinstitute.dsm.model.Filter; + +public class SplitterFactory { + + public static BaseSplitter createSplitter(String filter) { + BaseSplitter splitter; + if (filter.contains(Filter.EQUALS_TRIMMED)) + splitter = new EqualsSplitter(filter); + else if (filter.contains(Filter.LIKE_TRIMMED)) + splitter = new LikeSplitter(filter); + else if (filter.contains(Filter.LARGER_EQUALS_TRIMMED)) + splitter = new GreaterThanEqualsSplitter(filter); + else if (filter.contains(Filter.SMALLER_EQUALS_TRIMMED)) + splitter = new LessThanEqualsSplitter(filter); + else if (filter.contains(Filter.IS_NOT_NULL.trim())) + splitter = new IsNotNullSplitter(filter); + else + throw new IllegalArgumentException("Unknown operator"); + return splitter; + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Unit.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Unit.java deleted file mode 100644 index 532a3b5ec..000000000 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Unit.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.broadinstitute.dsm.model.elastic.filter; - -import java.util.List; -import java.util.Objects; - -public class Unit { - - protected List splittedWords; - protected String valueToSplit; - - public List getSplittedWords() { - return splittedWords; - } - - public void setSplittedWords(List splittedWords) { - this.splittedWords = splittedWords; - } - - public String getValueToSplit() { - return valueToSplit; - } - - public void setValueToSplit(String valueToSplit) { - this.valueToSplit = valueToSplit; - } - - - public Unit(String valueToSplit) { - this.valueToSplit = Objects.requireNonNull(valueToSplit); - } - - public void accept(SpliteratorVisitor visitor) { - visitor.visit(this); - } - -} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/VisitorFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/VisitorFactory.java deleted file mode 100644 index 54652dd07..000000000 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/VisitorFactory.java +++ /dev/null @@ -1,4 +0,0 @@ -package org.broadinstitute.dsm.model.elastic.filter; - -public class VisitorFactory { -} diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java index c8153607f..167888e38 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java @@ -121,4 +121,5 @@ public void camelCaseToPascalSnakeCase() { assertEquals("REGISTRATION_TYPE", pascalSnakeCase); assertEquals("TEST", pascalSnakeCase2); } + } \ No newline at end of file From 5d459bc2d59cd6bc1ff2d744e90c3c6ac07dfaf9 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 16 Dec 2021 16:09:10 +0400 Subject: [PATCH 229/405] DDP-7121 | add operator enum class --- .../filter/CollectionQueryBuilder.java | 5 ++++ .../dsm/model/elastic/filter/Operator.java | 26 +++++++++++++++++++ .../filter/CollectionQueryBuilderTest.java | 10 +++++++ .../model/elastic/filter/OperatorTest.java | 23 ++++++++++++++++ 4 files changed, 64 insertions(+) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java index b2890b18a..4cca0d4bf 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java @@ -15,6 +15,11 @@ public CollectionQueryBuilder(String filter) { super(filter); } + public static Operator extract(String filterEquals) { + String operator = filterEquals.split(" ")[1]; + return Operator.getOperator(operator); + } + @Override public AbstractQueryBuilder build() { Map> parsedFilters = parseFiltersByLogicalOperators(); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java new file mode 100644 index 000000000..6f29df3ac --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java @@ -0,0 +1,26 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import java.util.Arrays; + +import org.broadinstitute.dsm.model.Filter; + +public enum Operator { + LIKE(Filter.LIKE_TRIMMED), + EQUALS(Filter.EQUALS_TRIMMED), + GREATER_THAN_EQUALS(Filter.LARGER_EQUALS_TRIMMED), + LESS_THAN_EQUALS(Filter.SMALLER_EQUALS_TRIMMED), + IS_NOT_NULL(Filter.IS_NOT_NULL); + + private String value; + + Operator(String value) { + this.value = value; + } + + public static Operator getOperator(String value) { + for (Operator op: Operator.values()) { + if (op.value.equals(value)) return op; + } + throw new IllegalArgumentException("Unknown operator"); + } +} diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java index 3813e60ea..e5228c3c4 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java @@ -25,4 +25,14 @@ public void parseFiltersByLogicalOperators() { } } } + + @Test + public void extractLogicalOperator() { + String filterEquals = "m.medicalRecordId = 15"; + String filterLike = "m.medicalRecordId LIKE 15"; + Operator equalsOperator = CollectionQueryBuilder.extract(filterEquals); + Operator likeOperator = CollectionQueryBuilder.extract(filterLike); + assertEquals(Operator.EQUALS, equalsOperator); + assertEquals(Operator.LIKE, likeOperator); + } } \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java new file mode 100644 index 000000000..0c9ab04bd --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java @@ -0,0 +1,23 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import static org.junit.Assert.*; + +import org.broadinstitute.dsm.model.Filter; +import org.junit.Assert; +import org.junit.Test; + +public class OperatorTest { + + @Test + public void getOperator() { + try { + Operator like = Operator.getOperator(Filter.LIKE_TRIMMED); + Operator equals = Operator.getOperator(Filter.EQUALS_TRIMMED); + Operator unknown = Operator.getOperator(Filter.TODAY); + assertEquals(Operator.LIKE, like); + assertEquals(Operator.EQUALS, equals); + } catch (IllegalArgumentException iae) { + Assert.assertTrue(true); + } + } +} \ No newline at end of file From 7ac8a645ee65ec30e70258f452c5f6e06671feb7 Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 16 Dec 2021 16:42:22 +0400 Subject: [PATCH 230/405] DDP-7121 | create splitter based on Operator --- .../org/broadinstitute/dsm/model/Filter.java | 1 + .../model/elastic/filter/BaseSplitter.java | 10 +++--- .../filter/CollectionQueryBuilder.java | 18 +++++----- .../model/elastic/filter/EqualsSplitter.java | 4 --- .../filter/GreaterThanEqualsSplitter.java | 4 --- .../elastic/filter/IsNotNullSplitter.java | 4 --- .../filter/LessThanEqualsSplitter.java | 4 --- .../model/elastic/filter/LikeSplitter.java | 5 --- .../dsm/model/elastic/filter/Operator.java | 9 +++-- .../model/elastic/filter/SplitterFactory.java | 35 +++++++++++-------- .../filter/CollectionQueryBuilderTest.java | 10 ------ .../model/elastic/filter/OperatorTest.java | 16 +++++++++ 12 files changed, 56 insertions(+), 64 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/Filter.java b/src/main/java/org/broadinstitute/dsm/model/Filter.java index 8bb074af4..74c5a6999 100644 --- a/src/main/java/org/broadinstitute/dsm/model/Filter.java +++ b/src/main/java/org/broadinstitute/dsm/model/Filter.java @@ -54,6 +54,7 @@ public class Filter { public static final String JSON_CONTAINS = "JSON_CONTAINS"; public static final String JSON_OBJECT = "JSON_OBJECT"; public static final int THOUSAND = 1000; + public static final String IS_NOT_NULL_TRIMMED = "IS NOT NULL"; public static String TEXT = "TEXT"; public static String OPTIONS = "OPTIONS"; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitter.java index 1064510c4..316a9442c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitter.java @@ -7,11 +7,6 @@ public abstract class BaseSplitter { protected String filter; protected String[] splittedFilter; - public BaseSplitter(String filter) { - this.filter = filter; - splittedFilter = split(); - } - public abstract String[] split(); public String getValue() { @@ -29,4 +24,9 @@ public String getInnerProperty() { private String[] getFieldWithAlias() { return splittedFilter[0].trim().split(ElasticSearchUtil.DOT_SEPARATOR); } + + public void setFilter(String filter) { + this.filter = filter; + splittedFilter = split(); + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java index 4cca0d4bf..cba8549e2 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java @@ -1,13 +1,15 @@ package org.broadinstitute.dsm.model.elastic.filter; -import java.util.*; - import org.apache.commons.lang3.StringUtils; import org.apache.lucene.search.join.ScoreMode; import org.broadinstitute.dsm.model.Filter; import org.broadinstitute.dsm.model.elastic.Util; -import org.broadinstitute.dsm.util.ElasticSearchUtil; -import org.elasticsearch.index.query.*; +import org.elasticsearch.index.query.AbstractQueryBuilder; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.index.query.NestedQueryBuilder; + +import java.util.*; public class CollectionQueryBuilder extends DsmAbstractQueryBuilder { @@ -15,11 +17,6 @@ public CollectionQueryBuilder(String filter) { super(filter); } - public static Operator extract(String filterEquals) { - String operator = filterEquals.split(" ")[1]; - return Operator.getOperator(operator); - } - @Override public AbstractQueryBuilder build() { Map> parsedFilters = parseFiltersByLogicalOperators(); @@ -37,7 +34,8 @@ public AbstractQueryBuilder build() { private void buildUpNestedQuery(BoolQueryBuilder boolQueryBuilder, List filterValues, FilterStrategy filterStrategy) { for (String filterValue : filterValues) { - BaseSplitter splitter = SplitterFactory.createSplitter(filterValue); + BaseSplitter splitter = SplitterFactory.createSplitter(Operator.extract(filterValue)); + splitter.setFilter(filterValue); String outerProperty = Util.TABLE_ALIAS_MAPPINGS.get(splitter.getAlias()).getPropertyName(); //medicalRecord String nestedPath = DSM_WITH_DOT + outerProperty; filterStrategy.build(boolQueryBuilder, buildNestedQueryBuilder(nestedPath, splitter.getInnerProperty(), splitter.getValue())); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/EqualsSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/EqualsSplitter.java index dd8eaf239..2035999fd 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/EqualsSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/EqualsSplitter.java @@ -5,10 +5,6 @@ public class EqualsSplitter extends BaseSplitter { - public EqualsSplitter(String filter) { - super(filter); - } - @Override public String[] split() { return filter.split(Filter.EQUALS_TRIMMED); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/GreaterThanEqualsSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/GreaterThanEqualsSplitter.java index 98a859df8..c67f557ac 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/GreaterThanEqualsSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/GreaterThanEqualsSplitter.java @@ -4,10 +4,6 @@ public class GreaterThanEqualsSplitter extends BaseSplitter { - public GreaterThanEqualsSplitter(String filter) { - super(filter); - } - @Override public String[] split() { return filter.split(Filter.LARGER_EQUALS_TRIMMED); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/IsNotNullSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/IsNotNullSplitter.java index b61dbedad..ea42d5b20 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/IsNotNullSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/IsNotNullSplitter.java @@ -4,10 +4,6 @@ public class IsNotNullSplitter extends BaseSplitter { - public IsNotNullSplitter(String filter) { - super(filter); - } - @Override public String[] split() { return filter.split(Filter.IS_NOT_NULL.trim()); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LessThanEqualsSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LessThanEqualsSplitter.java index 487c5b100..0a718cf72 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LessThanEqualsSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LessThanEqualsSplitter.java @@ -4,10 +4,6 @@ public class LessThanEqualsSplitter extends BaseSplitter { - public LessThanEqualsSplitter(String filter) { - super(filter); - } - @Override public String[] split() { return filter.split(Filter.SMALLER_EQUALS_TRIMMED); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LikeSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LikeSplitter.java index c338f2663..4a9827048 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LikeSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LikeSplitter.java @@ -5,11 +5,6 @@ public class LikeSplitter extends BaseSplitter { - - public LikeSplitter(String filter) { - super(filter); - } - @Override public String[] split() { return filter.split(Filter.EQUALS_TRIMMED); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java index 6f29df3ac..c69ca7430 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java @@ -1,7 +1,5 @@ package org.broadinstitute.dsm.model.elastic.filter; -import java.util.Arrays; - import org.broadinstitute.dsm.model.Filter; public enum Operator { @@ -9,7 +7,7 @@ public enum Operator { EQUALS(Filter.EQUALS_TRIMMED), GREATER_THAN_EQUALS(Filter.LARGER_EQUALS_TRIMMED), LESS_THAN_EQUALS(Filter.SMALLER_EQUALS_TRIMMED), - IS_NOT_NULL(Filter.IS_NOT_NULL); + IS_NOT_NULL(Filter.IS_NOT_NULL_TRIMMED); private String value; @@ -23,4 +21,9 @@ public static Operator getOperator(String value) { } throw new IllegalArgumentException("Unknown operator"); } + + public static Operator extract(String filter) { + String operator = filter.split(" ")[1]; + return getOperator(operator); + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java index 441a96051..c66c1eb27 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java @@ -1,23 +1,28 @@ package org.broadinstitute.dsm.model.elastic.filter; -import org.broadinstitute.dsm.model.Filter; - public class SplitterFactory { - public static BaseSplitter createSplitter(String filter) { + public static BaseSplitter createSplitter(Operator operator) { BaseSplitter splitter; - if (filter.contains(Filter.EQUALS_TRIMMED)) - splitter = new EqualsSplitter(filter); - else if (filter.contains(Filter.LIKE_TRIMMED)) - splitter = new LikeSplitter(filter); - else if (filter.contains(Filter.LARGER_EQUALS_TRIMMED)) - splitter = new GreaterThanEqualsSplitter(filter); - else if (filter.contains(Filter.SMALLER_EQUALS_TRIMMED)) - splitter = new LessThanEqualsSplitter(filter); - else if (filter.contains(Filter.IS_NOT_NULL.trim())) - splitter = new IsNotNullSplitter(filter); - else - throw new IllegalArgumentException("Unknown operator"); + switch (operator) { + case EQUALS: + splitter = new EqualsSplitter(); + break; + case LIKE: + splitter = new LikeSplitter(); + break; + case GREATER_THAN_EQUALS: + splitter = new GreaterThanEqualsSplitter(); + break; + case LESS_THAN_EQUALS: + splitter = new LessThanEqualsSplitter(); + break; + case IS_NOT_NULL: + splitter = new IsNotNullSplitter(); + break; + default: + throw new IllegalArgumentException("Unknown operator"); + } return splitter; } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java index e5228c3c4..dd6e3b7ee 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java @@ -4,7 +4,6 @@ import org.junit.Test; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; @@ -26,13 +25,4 @@ public void parseFiltersByLogicalOperators() { } } - @Test - public void extractLogicalOperator() { - String filterEquals = "m.medicalRecordId = 15"; - String filterLike = "m.medicalRecordId LIKE 15"; - Operator equalsOperator = CollectionQueryBuilder.extract(filterEquals); - Operator likeOperator = CollectionQueryBuilder.extract(filterLike); - assertEquals(Operator.EQUALS, equalsOperator); - assertEquals(Operator.LIKE, likeOperator); - } } \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java index 0c9ab04bd..1c86cc0c0 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java @@ -13,11 +13,27 @@ public void getOperator() { try { Operator like = Operator.getOperator(Filter.LIKE_TRIMMED); Operator equals = Operator.getOperator(Filter.EQUALS_TRIMMED); + Operator larger = Operator.getOperator(Filter.LARGER_EQUALS_TRIMMED); + Operator smaller = Operator.getOperator(Filter.SMALLER_EQUALS_TRIMMED); + Operator isNotNull = Operator.getOperator(Filter.IS_NOT_NULL_TRIMMED); Operator unknown = Operator.getOperator(Filter.TODAY); assertEquals(Operator.LIKE, like); assertEquals(Operator.EQUALS, equals); + assertEquals(Operator.GREATER_THAN_EQUALS, larger); + assertEquals(Operator.LESS_THAN_EQUALS, smaller); + assertEquals(Operator.IS_NOT_NULL, isNotNull); } catch (IllegalArgumentException iae) { Assert.assertTrue(true); } } + + @Test + public void extractLogicalOperator() { + String filterEquals = "m.medicalRecordId = 15"; + String filterLike = "m.medicalRecordId LIKE 15"; + Operator equalsOperator = Operator.extract(filterEquals); + Operator likeOperator = Operator.extract(filterLike); + assertEquals(Operator.EQUALS, equalsOperator); + assertEquals(Operator.LIKE, likeOperator); + } } \ No newline at end of file From 4d24263f7a139eb63c6631eabd476983dff43cc3 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 16 Dec 2021 17:12:06 +0400 Subject: [PATCH 231/405] DDP-7121 | add query factory and query payload --- .../elastic/filter/QueryBuilderFactory.java | 28 +++++++++++++++++++ .../model/elastic/filter/QueryPayload.java | 18 ++++++++++++ .../filter/CollectionQueryBuilderTest.java | 4 +++ .../filter/QueryBuilderFactoryTest.java | 27 ++++++++++++++++++ 4 files changed, 77 insertions(+) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryPayload.java create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactoryTest.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java new file mode 100644 index 000000000..5dbf1bb12 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java @@ -0,0 +1,28 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import org.elasticsearch.index.query.ExistsQueryBuilder; +import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.RangeQueryBuilder; + +public class QueryBuilderFactory { + public static QueryBuilder buildQueryBuilder(Operator operator, QueryPayload payload) { + QueryBuilder queryBuilder; + switch (operator) { + case LIKE: + case EQUALS: + queryBuilder = new MatchQueryBuilder(payload.getFieldName(), payload.getValue()); + break; + case GREATER_THAN_EQUALS: + case LESS_THAN_EQUALS: + queryBuilder = new RangeQueryBuilder(payload.getFieldName()); + break; + case IS_NOT_NULL: + queryBuilder = new ExistsQueryBuilder(payload.getFieldName()); + break; + default: + throw new IllegalArgumentException("Unknown operator"); + } + return queryBuilder; + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryPayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryPayload.java new file mode 100644 index 000000000..cf84d7edd --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryPayload.java @@ -0,0 +1,18 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import lombok.Getter; + +@Getter +public class QueryPayload { + String fieldName; + Object value; + + public QueryPayload(String fieldName, Object value) { + this.fieldName = fieldName; + this.value = value; + } + + public QueryPayload(String fieldName) { + this.fieldName = fieldName; + } +} diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java index dd6e3b7ee..bedbbaa3e 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java @@ -1,5 +1,8 @@ package org.broadinstitute.dsm.model.elastic.filter; +import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.RangeQueryBuilder; import org.junit.Assert; import org.junit.Test; @@ -25,4 +28,5 @@ public void parseFiltersByLogicalOperators() { } } + } \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactoryTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactoryTest.java new file mode 100644 index 000000000..d14dbb4a4 --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactoryTest.java @@ -0,0 +1,27 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import static org.junit.Assert.*; + +import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.junit.Test; + +public class QueryBuilderFactoryTest { + + + @Test + public void buildQueryBuilder() { + QueryPayload payload = new QueryPayload("medicalRecordId", 15); + Operator operator = Operator.EQUALS; + QueryBuilder queryBuilder = QueryBuilderFactory.buildQueryBuilder(operator, payload); + assertTrue(queryBuilder instanceof MatchQueryBuilder); + operator = Operator.LIKE; + queryBuilder = QueryBuilderFactory.buildQueryBuilder(operator, payload); + assertTrue(queryBuilder instanceof MatchQueryBuilder); + operator = Operator.GREATER_THAN_EQUALS; + queryBuilder = QueryBuilderFactory.buildQueryBuilder(operator, payload); + assertTrue(queryBuilder instanceof RangeQueryBuilder); + } + +} \ No newline at end of file From 1e42c0dd297b1f16d97549fc65b29be421fd8f0d Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 16 Dec 2021 17:41:45 +0400 Subject: [PATCH 232/405] DDP-7121 | fix bugs based on unit tests --- .../model/elastic/filter/BaseSplitter.java | 6 ++- .../filter/CollectionQueryBuilder.java | 17 ++++---- .../elastic/filter/IsNotNullSplitter.java | 2 +- .../model/elastic/filter/LikeSplitter.java | 2 +- .../dsm/model/elastic/filter/Operator.java | 2 + .../elastic/filter/QueryBuilderFactory.java | 8 +++- .../filter/CollectionQueryBuilderTest.java | 40 +++++++++++++++++-- .../filter/DsmAbstractQueryBuilderTest.java | 35 ---------------- 8 files changed, 60 insertions(+), 52 deletions(-) delete mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilderTest.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitter.java index 316a9442c..d7335a9f7 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitter.java @@ -1,5 +1,6 @@ package org.broadinstitute.dsm.model.elastic.filter; +import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.util.ElasticSearchUtil; public abstract class BaseSplitter { @@ -10,7 +11,10 @@ public abstract class BaseSplitter { public abstract String[] split(); public String getValue() { - return splittedFilter[1].trim(); + if (splittedFilter.length > 1) { + return splittedFilter[1].trim(); + } + return StringUtils.EMPTY; } public String getAlias() { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java index cba8549e2..71a89ec2a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java @@ -1,13 +1,11 @@ package org.broadinstitute.dsm.model.elastic.filter; import org.apache.commons.lang3.StringUtils; +import org.apache.lucene.search.Query; import org.apache.lucene.search.join.ScoreMode; import org.broadinstitute.dsm.model.Filter; import org.broadinstitute.dsm.model.elastic.Util; -import org.elasticsearch.index.query.AbstractQueryBuilder; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.MatchQueryBuilder; -import org.elasticsearch.index.query.NestedQueryBuilder; +import org.elasticsearch.index.query.*; import java.util.*; @@ -34,18 +32,17 @@ public AbstractQueryBuilder build() { private void buildUpNestedQuery(BoolQueryBuilder boolQueryBuilder, List filterValues, FilterStrategy filterStrategy) { for (String filterValue : filterValues) { - BaseSplitter splitter = SplitterFactory.createSplitter(Operator.extract(filterValue)); + Operator operator = Operator.extract(filterValue); + BaseSplitter splitter = SplitterFactory.createSplitter(operator); splitter.setFilter(filterValue); String outerProperty = Util.TABLE_ALIAS_MAPPINGS.get(splitter.getAlias()).getPropertyName(); //medicalRecord String nestedPath = DSM_WITH_DOT + outerProperty; - filterStrategy.build(boolQueryBuilder, buildNestedQueryBuilder(nestedPath, splitter.getInnerProperty(), splitter.getValue())); + QueryPayload queryPayload = new QueryPayload(nestedPath + "." + splitter.getInnerProperty(), splitter.getValue()); + QueryBuilder queryBuilder = QueryBuilderFactory.buildQueryBuilder(operator, queryPayload); + filterStrategy.build(boolQueryBuilder, new NestedQueryBuilder(nestedPath, queryBuilder, ScoreMode.Avg)); } } - private NestedQueryBuilder buildNestedQueryBuilder(String path, String fieldName, Object value) { - return new NestedQueryBuilder(path, new MatchQueryBuilder(path + "." + fieldName, value), ScoreMode.Avg); - } - protected Map> parseFiltersByLogicalOperators() { Map> filterByLogicalOperators = new HashMap<>(Map.of(Filter.AND, new ArrayList<>(), Filter.OR, new ArrayList<>())); String[] andSeparated = filter.split(Filter.AND_TRIMMED); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/IsNotNullSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/IsNotNullSplitter.java index ea42d5b20..888fbb8bb 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/IsNotNullSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/IsNotNullSplitter.java @@ -6,6 +6,6 @@ public class IsNotNullSplitter extends BaseSplitter { @Override public String[] split() { - return filter.split(Filter.IS_NOT_NULL.trim()); + return filter.split(Filter.IS_NOT_NULL_TRIMMED); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LikeSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LikeSplitter.java index 4a9827048..3f34ae3f8 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LikeSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LikeSplitter.java @@ -7,6 +7,6 @@ public class LikeSplitter extends BaseSplitter { @Override public String[] split() { - return filter.split(Filter.EQUALS_TRIMMED); + return filter.split(Filter.LIKE_TRIMMED); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java index c69ca7430..39e58359c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java @@ -23,6 +23,8 @@ public static Operator getOperator(String value) { } public static Operator extract(String filter) { + if (filter.contains(Filter.IS_NOT_NULL_TRIMMED)) + return IS_NOT_NULL; String operator = filter.split(" ")[1]; return getOperator(operator); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java index 5dbf1bb12..7c0acef01 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java @@ -14,8 +14,14 @@ public static QueryBuilder buildQueryBuilder(Operator operator, QueryPayload pay queryBuilder = new MatchQueryBuilder(payload.getFieldName(), payload.getValue()); break; case GREATER_THAN_EQUALS: + RangeQueryBuilder greaterRangeQuery = new RangeQueryBuilder(payload.getFieldName()); + greaterRangeQuery.gte(payload.getValue()); + queryBuilder = greaterRangeQuery; + break; case LESS_THAN_EQUALS: - queryBuilder = new RangeQueryBuilder(payload.getFieldName()); + RangeQueryBuilder lessRangeQuery = new RangeQueryBuilder(payload.getFieldName()); + lessRangeQuery.lte(payload.getValue()); + queryBuilder = lessRangeQuery; break; case IS_NOT_NULL: queryBuilder = new ExistsQueryBuilder(payload.getFieldName()); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java index bedbbaa3e..ac98dcf56 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java @@ -1,8 +1,7 @@ package org.broadinstitute.dsm.model.elastic.filter; -import org.elasticsearch.index.query.MatchQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.RangeQueryBuilder; +import org.apache.lucene.search.join.ScoreMode; +import org.elasticsearch.index.query.*; import org.junit.Assert; import org.junit.Test; @@ -28,5 +27,40 @@ public void parseFiltersByLogicalOperators() { } } + @Test + public void collectionBuild() { + + String filter = "AND m.medicalRecordId = 15 AND m.type = PHYSICIAN OR k.bspCollaboratorSampleId = ASCProject_PZ8GJC_SALIVA"; + + DsmAbstractQueryBuilder dsmAbstractQueryBuilder = new CollectionQueryBuilder(filter); + + AbstractQueryBuilder actual = dsmAbstractQueryBuilder.build(); + + AbstractQueryBuilder expected = new BoolQueryBuilder().must(new NestedQueryBuilder("dsm.medicalRecord", new MatchQueryBuilder("dsm.medicalRecord.medicalRecordId", "15"), ScoreMode.Avg)) + .must(new NestedQueryBuilder("dsm.medicalRecord", new MatchQueryBuilder("dsm.medicalRecord.type", "PHYSICIAN"), ScoreMode.Avg)) + .should(new NestedQueryBuilder("dsm.kitRequestShipping", new MatchQueryBuilder("dsm.kitRequestShipping.bspCollaboratorSampleId", "ASCProject_PZ8GJC_SALIVA"), ScoreMode.Avg)); + + Assert.assertEquals(expected, actual); + } + + @Test + public void collectionBuild2() { + + String filter = "AND m.medicalRecordId >= 15 AND m.type LIKE PHYSICIAN OR k.bspCollaboratorSampleId = ASCProject_PZ8GJC_SALIVA AND t.returnDate <= 2015-01-01 AND p.participantId IS NOT NULL"; + + DsmAbstractQueryBuilder dsmAbstractQueryBuilder = new CollectionQueryBuilder(filter); + + AbstractQueryBuilder actual = dsmAbstractQueryBuilder.build(); + + AbstractQueryBuilder expected = new BoolQueryBuilder().must(new NestedQueryBuilder("dsm.medicalRecord", new RangeQueryBuilder("dsm.medicalRecord.medicalRecordId").gte("15"), ScoreMode.Avg)) + .must(new NestedQueryBuilder("dsm.medicalRecord", new MatchQueryBuilder("dsm.medicalRecord.type", "PHYSICIAN"), ScoreMode.Avg)) + .should(new NestedQueryBuilder("dsm.kitRequestShipping", new MatchQueryBuilder("dsm.kitRequestShipping.bspCollaboratorSampleId", "ASCProject_PZ8GJC_SALIVA"), ScoreMode.Avg)) + .must(new NestedQueryBuilder("dsm.tissue", new RangeQueryBuilder("dsm.tissue.returnDate").lte("2015-01-01"), ScoreMode.Avg)) + .must(new NestedQueryBuilder("dsm.participant", new ExistsQueryBuilder("dsm.participant.participantId"), ScoreMode.Avg)); + + Assert.assertEquals(expected, actual); + } + + } \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilderTest.java deleted file mode 100644 index 46b370b1a..000000000 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilderTest.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.broadinstitute.dsm.model.elastic.filter; - -import org.apache.lucene.search.join.ScoreMode; -import org.elasticsearch.index.query.AbstractQueryBuilder; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.MatchQueryBuilder; -import org.elasticsearch.index.query.NestedQueryBuilder; -import org.junit.Assert; -import org.junit.Test; - -public class DsmAbstractQueryBuilderTest { - - @Test - public void collectionBuild() { - - String filter = "AND m.medicalRecordId = 15 AND m.type = PHYSICIAN OR k.bspCollaboratorSampleId = ASCProject_PZ8GJC_SALIVA"; - - DsmAbstractQueryBuilder dsmAbstractQueryBuilder = new CollectionQueryBuilder(filter); - - AbstractQueryBuilder actual = dsmAbstractQueryBuilder.build(); - - AbstractQueryBuilder expected = new BoolQueryBuilder().must(new NestedQueryBuilder("dsm.medicalRecord", new MatchQueryBuilder("dsm.medicalRecord.medicalRecordId", "15"), ScoreMode.Avg)) - .must(new NestedQueryBuilder("dsm.medicalRecord", new MatchQueryBuilder("dsm.medicalRecord.type", "PHYSICIAN"), ScoreMode.Avg)) - .should(new NestedQueryBuilder("dsm.kitRequestShipping", new MatchQueryBuilder("dsm.kitRequestShipping.bspCollaboratorSampleId", "ASCProject_PZ8GJC_SALIVA"), ScoreMode.Avg)); - - Assert.assertEquals(expected, actual); - } - - @Test - public void singleBuild() { - - - } - -} From 526394a2cd2c7e2a27d5b28cb204057b1e0682a2 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 17 Dec 2021 10:47:51 +0400 Subject: [PATCH 233/405] DDP-7121 | add test and method to parse '0'/'1' values to boolean --- .../elastic/export/parse/BaseParser.java | 8 +++- .../elastic/export/parse/ValueParser.java | 1 + .../filter/CollectionQueryBuilder.java | 13 +++--- .../elastic/export/parse/BaseParserTest.java | 28 ++++++++++++ .../filter/CollectionQueryBuilderTest.java | 43 ++++++++++++++++--- 5 files changed, 81 insertions(+), 12 deletions(-) create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserTest.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java index 0ce01fd82..48bf02cde 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java @@ -1,9 +1,12 @@ package org.broadinstitute.dsm.model.elastic.export.parse; +import java.nio.CharBuffer; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; import java.time.format.DateTimeParseException; +import java.util.Arrays; +import java.util.stream.IntStream; import org.apache.commons.lang3.StringUtils; @@ -62,7 +65,10 @@ boolean isDate(String value) { } boolean isBoolean(String value) { + IntStream chars = CharBuffer.wrap(value.toCharArray()).chars(); + boolean isBoolean = chars.anyMatch(c -> ((char) c) == '1' || ((char) c) == '0'); return value.equalsIgnoreCase(Boolean.TRUE.toString()) || - value.equalsIgnoreCase(Boolean.FALSE.toString()); + value.equalsIgnoreCase(Boolean.FALSE.toString()) || + isBoolean ; } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/ValueParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/ValueParser.java index 10fd5ceb3..e09622007 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/ValueParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/ValueParser.java @@ -9,6 +9,7 @@ protected Object forNumeric(String value) { @Override protected Object forBoolean(String value) { + return Boolean.valueOf(value); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java index 71a89ec2a..fdaec13fb 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java @@ -44,18 +44,19 @@ private void buildUpNestedQuery(BoolQueryBuilder boolQueryBuilder, List } protected Map> parseFiltersByLogicalOperators() { - Map> filterByLogicalOperators = new HashMap<>(Map.of(Filter.AND, new ArrayList<>(), Filter.OR, new ArrayList<>())); + Map> filterByLogicalOperators = new HashMap<>(Map.of(Filter.AND_TRIMMED, new ArrayList<>(), Filter.OR_TRIMMED, + new ArrayList<>())); String[] andSeparated = filter.split(Filter.AND_TRIMMED); for (String eachFilter : andSeparated) { String cleanedEachFilter = eachFilter.trim(); - if (cleanedEachFilter.contains(Filter.OR)) { - String[] orSeparated = cleanedEachFilter.split(Filter.OR); - filterByLogicalOperators.get(Filter.AND).add(orSeparated[0].trim()); + if (cleanedEachFilter.contains(Filter.OR_TRIMMED)) { + String[] orSeparated = cleanedEachFilter.split(Filter.OR_TRIMMED); + filterByLogicalOperators.get(Filter.AND_TRIMMED).add(orSeparated[0].trim()); Arrays.stream(orSeparated) .skip(1) - .forEach(f -> filterByLogicalOperators.get(Filter.OR).add(f.trim())); + .forEach(f -> filterByLogicalOperators.get(Filter.OR_TRIMMED).add(f.trim())); } else if (StringUtils.isNotBlank(cleanedEachFilter)){ - filterByLogicalOperators.get(Filter.AND).add(cleanedEachFilter); + filterByLogicalOperators.get(Filter.AND_TRIMMED).add(cleanedEachFilter); } } return filterByLogicalOperators; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserTest.java new file mode 100644 index 000000000..d4c22970a --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserTest.java @@ -0,0 +1,28 @@ +package org.broadinstitute.dsm.model.elastic.export.parse; + +import static org.junit.Assert.*; + +import org.junit.Test; + +public class BaseParserTest { + + @Test + public void isBoolean() { + String falseValue = "'0'"; + String trueValue = "'1'"; + BaseParser valueParser = new ValueParser(); + assertTrue(valueParser.isBoolean(falseValue)); + assertTrue(valueParser.isBoolean(trueValue)); + } + + @Test + public void convertBoolean() { + String falseValue = "'0'"; + String trueValue = "'1'"; + BaseParser valueParser = new ValueParser(); + boolean falseBool = valueParser.convertBoolean(falseValue); + boolean trueBool = valueParser.convertBoolean(falseValue); + assertTrue(trueBool); + assertFalse(falseBool); + } +} \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java index ac98dcf56..fbd5a8f40 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java @@ -15,14 +15,15 @@ public class CollectionQueryBuilderTest { @Test public void parseFiltersByLogicalOperators() { - String filter = "AND m.medicalRecordId = 15 OR m.medicalRecordSomething LIKE 55555 OR m.medicalRecordSomethingg = 55552 AND m.dynamicFields.ragac = 55 OR m.medicalRecordName = 213"; + String filter = "AND m.medicalRecordId = '15' OR m.medicalRecordSomething LIKE '55555' OR m.medicalRecordSomethingg = '55552' AND m.dynamicFields.ragac = '55' OR m.medicalRecordName = '213'"; CollectionQueryBuilder collectionQueryBuilder = new CollectionQueryBuilder(filter); Map> parsedFilters = collectionQueryBuilder.parseFiltersByLogicalOperators(); for (Map.Entry> eachFilter: parsedFilters.entrySet()) { if (eachFilter.getKey().equals("AND")) { - Assert.assertArrayEquals(new ArrayList<>(List.of("m.medicalRecordId = 15", "m.dynamicFields.ragac = 55")).toArray(), eachFilter.getValue().toArray()); + Assert.assertArrayEquals(new ArrayList<>(List.of("m.medicalRecordId = '15'", "m.dynamicFields.ragac = '55'")).toArray(), + eachFilter.getValue().toArray()); } else { - Assert.assertArrayEquals(new ArrayList<>(List.of("m.medicalRecordSomething LIKE 55555", "m.medicalRecordSomethingg = 55552", "m.medicalRecordName = 213")).toArray(), eachFilter.getValue().toArray()); + Assert.assertArrayEquals(new ArrayList<>(List.of("m.medicalRecordSomething LIKE '55555'", "m.medicalRecordSomethingg = '55552'", "m.medicalRecordName = '213'")).toArray(), eachFilter.getValue().toArray()); } } } @@ -30,7 +31,7 @@ public void parseFiltersByLogicalOperators() { @Test public void collectionBuild() { - String filter = "AND m.medicalRecordId = 15 AND m.type = PHYSICIAN OR k.bspCollaboratorSampleId = ASCProject_PZ8GJC_SALIVA"; + String filter = "AND m.medicalRecordId = '15' AND m.type = 'PHYSICIAN' OR k.bspCollaboratorSampleId = 'ASCProject_PZ8GJC_SALIVA'"; DsmAbstractQueryBuilder dsmAbstractQueryBuilder = new CollectionQueryBuilder(filter); @@ -46,7 +47,7 @@ public void collectionBuild() { @Test public void collectionBuild2() { - String filter = "AND m.medicalRecordId >= 15 AND m.type LIKE PHYSICIAN OR k.bspCollaboratorSampleId = ASCProject_PZ8GJC_SALIVA AND t.returnDate <= 2015-01-01 AND p.participantId IS NOT NULL"; + String filter = "AND m.medicalRecordId >= '15' AND m.type LIKE 'PHYSICIAN' OR k.bspCollaboratorSampleId = 'ASCProject_PZ8GJC_SALIVA' AND t.returnDate <= '2015-01-01' AND p.participantId IS NOT NULL"; DsmAbstractQueryBuilder dsmAbstractQueryBuilder = new CollectionQueryBuilder(filter); @@ -61,6 +62,38 @@ public void collectionBuild2() { Assert.assertEquals(expected, actual); } + @Test + public void collectionBuildAgeRange() { + + String filter = "AND m.age >= '15' AND m.age <= '30'"; + + DsmAbstractQueryBuilder dsmAbstractQueryBuilder = new CollectionQueryBuilder(filter); + + AbstractQueryBuilder actual = dsmAbstractQueryBuilder.build(); + + AbstractQueryBuilder expected = new BoolQueryBuilder().must(new NestedQueryBuilder("dsm.medicalRecord", + new RangeQueryBuilder("dsm.medicalRecord.age").gte("15"), ScoreMode.Avg)) + .must(new NestedQueryBuilder("dsm.medicalRecord", new RangeQueryBuilder("dsm.medicalRecord.age").lte("30"), + ScoreMode.Avg)); + + Assert.assertEquals(expected, actual); + } + + @Test + public void collectionBuildBoolean() { + + String filter = "AND m.followUp LIKE '1'"; + + DsmAbstractQueryBuilder dsmAbstractQueryBuilder = new CollectionQueryBuilder(filter); + + AbstractQueryBuilder actual = dsmAbstractQueryBuilder.build(); + + AbstractQueryBuilder expected = new BoolQueryBuilder().must(new NestedQueryBuilder("dsm.medicalRecord", + new MatchQueryBuilder("dsm.medicalRecord.followUp", "'1'"), ScoreMode.Avg)); + + Assert.assertEquals(expected, actual); + } + } \ No newline at end of file From 54eef14e714280ff1ca75fbfa15e35742f981bd1 Mon Sep 17 00:00:00 2001 From: Nika Date: Fri, 17 Dec 2021 11:18:18 +0400 Subject: [PATCH 234/405] DDP-7121 | add convertBoolean() method to BaseParser, make unit tests pass --- .../elastic/export/parse/BaseParser.java | 32 ++++++++++++------- .../elastic/export/parse/ValueParser.java | 5 +-- .../elastic/export/parse/BaseParserTest.java | 23 +++++++------ 3 files changed, 36 insertions(+), 24 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java index 48bf02cde..d8e1bc94a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java @@ -1,14 +1,11 @@ package org.broadinstitute.dsm.model.elastic.export.parse; -import java.nio.CharBuffer; +import org.apache.commons.lang3.StringUtils; + import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; import java.time.format.DateTimeParseException; -import java.util.Arrays; -import java.util.stream.IntStream; - -import org.apache.commons.lang3.StringUtils; public abstract class BaseParser implements Parser { @@ -18,7 +15,7 @@ public Object parse(String value) { if (StringUtils.isNumeric(value)) { result = forNumeric(value); } else if (isBoolean(value)) { - result = forBoolean(value); + result = forBoolean(convertBoolean(value)); } else if (isDateOrTimeOrDateTime(value)) { result = forDate(value); } @@ -65,10 +62,23 @@ boolean isDate(String value) { } boolean isBoolean(String value) { - IntStream chars = CharBuffer.wrap(value.toCharArray()).chars(); - boolean isBoolean = chars.anyMatch(c -> ((char) c) == '1' || ((char) c) == '0'); - return value.equalsIgnoreCase(Boolean.TRUE.toString()) || - value.equalsIgnoreCase(Boolean.FALSE.toString()) || - isBoolean ; + return convertBoolean(value).equalsIgnoreCase(Boolean.TRUE.toString()) || + convertBoolean(value).equalsIgnoreCase(Boolean.FALSE.toString()); + } + + public String convertBoolean(String value) { + if ("'1'".equals(value)) { + return "true"; + } else if ("'0'".equals(value)){ + return "false"; + } else { + return value; + } + } + + public String convertString(String value) { + if (value.charAt(0) == '\'' && value.charAt(value.length() - 1) == '\'') + return value.substring(1, value.length() - 1); + return value; } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/ValueParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/ValueParser.java index e09622007..46d3a2bd0 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/ValueParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/ValueParser.java @@ -8,10 +8,7 @@ protected Object forNumeric(String value) { } @Override - protected Object forBoolean(String value) { - - return Boolean.valueOf(value); - } + protected Object forBoolean(String value) { return Boolean.valueOf(value); } @Override protected Object forDate(String value) { diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserTest.java index d4c22970a..f29fe8a6c 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserTest.java @@ -2,27 +2,32 @@ import static org.junit.Assert.*; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; public class BaseParserTest { + static BaseParser valueParser; + + @BeforeClass + public static void setUp() { + valueParser = new ValueParser(); + } + @Test public void isBoolean() { String falseValue = "'0'"; String trueValue = "'1'"; - BaseParser valueParser = new ValueParser(); assertTrue(valueParser.isBoolean(falseValue)); assertTrue(valueParser.isBoolean(trueValue)); } @Test - public void convertBoolean() { - String falseValue = "'0'"; - String trueValue = "'1'"; - BaseParser valueParser = new ValueParser(); - boolean falseBool = valueParser.convertBoolean(falseValue); - boolean trueBool = valueParser.convertBoolean(falseValue); - assertTrue(trueBool); - assertFalse(falseBool); + public void parse() { + String value = "'15'"; + String convertedValue = valueParser.convertString(value); + Assert.assertEquals("15", convertedValue); } } \ No newline at end of file From af0116c10f937278d52ec5cb7ba7786aa3988634 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 17 Dec 2021 15:09:47 +0400 Subject: [PATCH 235/405] DDP-7121 | add filter parser to handle numeric values in the same manner as text values --- .../elastic/export/parse/BaseParser.java | 7 ++- .../filter/CollectionQueryBuilder.java | 12 +++-- .../filter/DsmAbstractQueryBuilder.java | 15 +++++- .../model/elastic/filter/FilterParser.java | 11 +++++ .../elastic/export/parse/BaseParserTest.java | 6 +++ .../filter/CollectionQueryBuilderTest.java | 48 +++++++++++-------- 6 files changed, 72 insertions(+), 27 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/FilterParser.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java index d8e1bc94a..e45e2052e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java @@ -11,6 +11,7 @@ public abstract class BaseParser implements Parser { @Override public Object parse(String value) { + value = convertString(value); Object result = forString(value); if (StringUtils.isNumeric(value)) { result = forNumeric(value); @@ -77,8 +78,12 @@ public String convertBoolean(String value) { } public String convertString(String value) { - if (value.charAt(0) == '\'' && value.charAt(value.length() - 1) == '\'') + if (isWrappedByChar(value)) return value.substring(1, value.length() - 1); return value; } + + private boolean isWrappedByChar(String value) { + return StringUtils.isNotBlank(value) && value.charAt(0) == '\'' && value.charAt(value.length() - 1) == '\''; + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java index fdaec13fb..b23cdcdb8 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java @@ -1,27 +1,29 @@ package org.broadinstitute.dsm.model.elastic.filter; import org.apache.commons.lang3.StringUtils; -import org.apache.lucene.search.Query; import org.apache.lucene.search.join.ScoreMode; import org.broadinstitute.dsm.model.Filter; import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import org.elasticsearch.index.query.*; import java.util.*; public class CollectionQueryBuilder extends DsmAbstractQueryBuilder { - public CollectionQueryBuilder(String filter) { - super(filter); + public CollectionQueryBuilder(String filter, Parser parser) { + super(filter, parser); } + public CollectionQueryBuilder() {} + @Override public AbstractQueryBuilder build() { Map> parsedFilters = parseFiltersByLogicalOperators(); BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); for (Map.Entry> parsedFilter: parsedFilters.entrySet()) { List filterValues = parsedFilter.getValue(); - if (parsedFilter.getKey().equals(Filter.AND)) { + if (parsedFilter.getKey().equals(Filter.AND_TRIMMED)) { buildUpNestedQuery(boolQueryBuilder, filterValues, BoolQueryBuilder::must); } else { buildUpNestedQuery(boolQueryBuilder, filterValues, BoolQueryBuilder::should); @@ -37,7 +39,7 @@ private void buildUpNestedQuery(BoolQueryBuilder boolQueryBuilder, List splitter.setFilter(filterValue); String outerProperty = Util.TABLE_ALIAS_MAPPINGS.get(splitter.getAlias()).getPropertyName(); //medicalRecord String nestedPath = DSM_WITH_DOT + outerProperty; - QueryPayload queryPayload = new QueryPayload(nestedPath + "." + splitter.getInnerProperty(), splitter.getValue()); + QueryPayload queryPayload = new QueryPayload(nestedPath + "." + splitter.getInnerProperty(), parser.parse(splitter.getValue())); QueryBuilder queryBuilder = QueryBuilderFactory.buildQueryBuilder(operator, queryPayload); filterStrategy.build(boolQueryBuilder, new NestedQueryBuilder(nestedPath, queryBuilder, ScoreMode.Avg)); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java index f738bfdeb..8d2f6e2dc 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java @@ -1,5 +1,6 @@ package org.broadinstitute.dsm.model.elastic.filter; +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.statics.ESObjectConstants; import org.elasticsearch.index.query.AbstractQueryBuilder; @@ -8,10 +9,22 @@ public abstract class DsmAbstractQueryBuilder { protected static final String DSM_WITH_DOT = ESObjectConstants.DSM + DBConstants.ALIAS_DELIMITER; protected String filter; + protected Parser parser; - public DsmAbstractQueryBuilder(String filter) { + public DsmAbstractQueryBuilder(String filter, Parser parser) { + this.parser = parser; this.filter = filter; } + public DsmAbstractQueryBuilder() {} + + public void setFilter(String filter) { + this.filter = filter; + } + + public void setParser(Parser parser) { + this.parser = parser; + } + public abstract AbstractQueryBuilder build(); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/FilterParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/FilterParser.java new file mode 100644 index 000000000..ac4b7f9cc --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/FilterParser.java @@ -0,0 +1,11 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import org.broadinstitute.dsm.model.elastic.export.parse.ValueParser; + +public class FilterParser extends ValueParser { + + @Override + protected Object forNumeric(String value) { + return value; + } +} diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserTest.java index f29fe8a6c..fe8a2643f 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserTest.java @@ -27,7 +27,13 @@ public void isBoolean() { @Test public void parse() { String value = "'15'"; + String value1 = "'ASCProject_PZ8GJC_SALIVA'"; + String value2 = "'2015-01-01'"; String convertedValue = valueParser.convertString(value); + String convertedValue1 = valueParser.convertString(value1); + String convertedValue2 = valueParser.convertString(value2); Assert.assertEquals("15", convertedValue); + Assert.assertEquals("ASCProject_PZ8GJC_SALIVA", convertedValue1); + Assert.assertEquals("2015-01-01", convertedValue2); } } \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java index fbd5a8f40..8fc58f768 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java @@ -1,22 +1,34 @@ package org.broadinstitute.dsm.model.elastic.filter; -import org.apache.lucene.search.join.ScoreMode; -import org.elasticsearch.index.query.*; -import org.junit.Assert; -import org.junit.Test; - import java.util.ArrayList; import java.util.List; import java.util.Map; -import static org.junit.Assert.*; +import org.apache.lucene.search.join.ScoreMode; +import org.elasticsearch.index.query.AbstractQueryBuilder; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.ExistsQueryBuilder; +import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.index.query.NestedQueryBuilder; +import org.elasticsearch.index.query.RangeQueryBuilder; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; public class CollectionQueryBuilderTest { + static CollectionQueryBuilder collectionQueryBuilder; + + @BeforeClass + public static void setUp() { + collectionQueryBuilder = new CollectionQueryBuilder(); + collectionQueryBuilder.setParser(new FilterParser()); + } + @Test public void parseFiltersByLogicalOperators() { String filter = "AND m.medicalRecordId = '15' OR m.medicalRecordSomething LIKE '55555' OR m.medicalRecordSomethingg = '55552' AND m.dynamicFields.ragac = '55' OR m.medicalRecordName = '213'"; - CollectionQueryBuilder collectionQueryBuilder = new CollectionQueryBuilder(filter); + collectionQueryBuilder.setFilter(filter); Map> parsedFilters = collectionQueryBuilder.parseFiltersByLogicalOperators(); for (Map.Entry> eachFilter: parsedFilters.entrySet()) { if (eachFilter.getKey().equals("AND")) { @@ -33,9 +45,8 @@ public void collectionBuild() { String filter = "AND m.medicalRecordId = '15' AND m.type = 'PHYSICIAN' OR k.bspCollaboratorSampleId = 'ASCProject_PZ8GJC_SALIVA'"; - DsmAbstractQueryBuilder dsmAbstractQueryBuilder = new CollectionQueryBuilder(filter); - - AbstractQueryBuilder actual = dsmAbstractQueryBuilder.build(); + collectionQueryBuilder.setFilter(filter); + AbstractQueryBuilder actual = collectionQueryBuilder.build(); AbstractQueryBuilder expected = new BoolQueryBuilder().must(new NestedQueryBuilder("dsm.medicalRecord", new MatchQueryBuilder("dsm.medicalRecord.medicalRecordId", "15"), ScoreMode.Avg)) .must(new NestedQueryBuilder("dsm.medicalRecord", new MatchQueryBuilder("dsm.medicalRecord.type", "PHYSICIAN"), ScoreMode.Avg)) @@ -49,9 +60,8 @@ public void collectionBuild2() { String filter = "AND m.medicalRecordId >= '15' AND m.type LIKE 'PHYSICIAN' OR k.bspCollaboratorSampleId = 'ASCProject_PZ8GJC_SALIVA' AND t.returnDate <= '2015-01-01' AND p.participantId IS NOT NULL"; - DsmAbstractQueryBuilder dsmAbstractQueryBuilder = new CollectionQueryBuilder(filter); - - AbstractQueryBuilder actual = dsmAbstractQueryBuilder.build(); + collectionQueryBuilder.setFilter(filter); + AbstractQueryBuilder actual = collectionQueryBuilder.build(); AbstractQueryBuilder expected = new BoolQueryBuilder().must(new NestedQueryBuilder("dsm.medicalRecord", new RangeQueryBuilder("dsm.medicalRecord.medicalRecordId").gte("15"), ScoreMode.Avg)) .must(new NestedQueryBuilder("dsm.medicalRecord", new MatchQueryBuilder("dsm.medicalRecord.type", "PHYSICIAN"), ScoreMode.Avg)) @@ -67,9 +77,8 @@ public void collectionBuildAgeRange() { String filter = "AND m.age >= '15' AND m.age <= '30'"; - DsmAbstractQueryBuilder dsmAbstractQueryBuilder = new CollectionQueryBuilder(filter); - - AbstractQueryBuilder actual = dsmAbstractQueryBuilder.build(); + collectionQueryBuilder.setFilter(filter); + AbstractQueryBuilder actual = collectionQueryBuilder.build(); AbstractQueryBuilder expected = new BoolQueryBuilder().must(new NestedQueryBuilder("dsm.medicalRecord", new RangeQueryBuilder("dsm.medicalRecord.age").gte("15"), ScoreMode.Avg)) @@ -84,12 +93,11 @@ public void collectionBuildBoolean() { String filter = "AND m.followUp LIKE '1'"; - DsmAbstractQueryBuilder dsmAbstractQueryBuilder = new CollectionQueryBuilder(filter); - - AbstractQueryBuilder actual = dsmAbstractQueryBuilder.build(); + collectionQueryBuilder.setFilter(filter); + AbstractQueryBuilder actual = collectionQueryBuilder.build(); AbstractQueryBuilder expected = new BoolQueryBuilder().must(new NestedQueryBuilder("dsm.medicalRecord", - new MatchQueryBuilder("dsm.medicalRecord.followUp", "'1'"), ScoreMode.Avg)); + new MatchQueryBuilder("dsm.medicalRecord.followUp", "1"), ScoreMode.Avg)); Assert.assertEquals(expected, actual); } From ebd4712415116714039650965222bfae903280ed Mon Sep 17 00:00:00 2001 From: Nika Date: Fri, 17 Dec 2021 16:12:06 +0400 Subject: [PATCH 236/405] DDP-7121 | override parse method from BaseParser to FilterParser, make unit tests pass --- .../elastic/export/parse/BaseParser.java | 2 +- .../model/elastic/filter/FilterParser.java | 8 +++++++ .../elastic/export/parse/BaseParserTest.java | 3 ++- .../elastic/filter/FilterParserTest.java | 24 +++++++++++++++++++ 4 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/filter/FilterParserTest.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java index e45e2052e..31dc84ab4 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java @@ -62,7 +62,7 @@ boolean isDate(String value) { return true; } - boolean isBoolean(String value) { + protected boolean isBoolean(String value) { return convertBoolean(value).equalsIgnoreCase(Boolean.TRUE.toString()) || convertBoolean(value).equalsIgnoreCase(Boolean.FALSE.toString()); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/FilterParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/FilterParser.java index ac4b7f9cc..87a66e242 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/FilterParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/FilterParser.java @@ -4,6 +4,14 @@ public class FilterParser extends ValueParser { + @Override + public Object parse(String value) { + if (isBoolean(value)) + return forBoolean(convertBoolean(value)); + else + return convertString(value); + } + @Override protected Object forNumeric(String value) { return value; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserTest.java index fe8a2643f..c0baeeb32 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserTest.java @@ -25,7 +25,7 @@ public void isBoolean() { } @Test - public void parse() { + public void convertString() { String value = "'15'"; String value1 = "'ASCProject_PZ8GJC_SALIVA'"; String value2 = "'2015-01-01'"; @@ -36,4 +36,5 @@ public void parse() { Assert.assertEquals("ASCProject_PZ8GJC_SALIVA", convertedValue1); Assert.assertEquals("2015-01-01", convertedValue2); } + } \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/FilterParserTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/FilterParserTest.java new file mode 100644 index 000000000..be40ef20d --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/FilterParserTest.java @@ -0,0 +1,24 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import org.broadinstitute.dsm.model.elastic.export.parse.BaseParser; +import org.junit.Assert; +import org.junit.Test; + +public class FilterParserTest { + + @Test + public void parse() { + + String trueValue = "'1'"; + String str = "'string'"; + String number = "'5'"; + String date = "'1999-05-22'"; + + BaseParser filterParser = new FilterParser(); + Assert.assertEquals(true, filterParser.parse(trueValue)); + Assert.assertEquals("string", filterParser.parse(str)); + Assert.assertEquals("5", filterParser.parse(number)); + Assert.assertEquals("1999-05-22", filterParser.parse(date)); + } + +} \ No newline at end of file From 7b590eef4db1343027abcd82dacdd373aabc1df6 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 17 Dec 2021 17:15:40 +0400 Subject: [PATCH 237/405] DDP-7121 | add diamond equals case --- .../org/broadinstitute/dsm/model/Filter.java | 1 + .../model/elastic/filter/BaseSplitter.java | 2 +- .../elastic/filter/DiamondEqualsSplitter.java | 19 ++++++++++++ .../dsm/model/elastic/filter/Operator.java | 3 +- .../model/elastic/filter/SplitterFactory.java | 3 ++ .../elastic/filter/BaseSplitterTest.java | 29 +++++++++++++++++++ 6 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitter.java create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitterTest.java diff --git a/src/main/java/org/broadinstitute/dsm/model/Filter.java b/src/main/java/org/broadinstitute/dsm/model/Filter.java index 74c5a6999..be0dfde98 100644 --- a/src/main/java/org/broadinstitute/dsm/model/Filter.java +++ b/src/main/java/org/broadinstitute/dsm/model/Filter.java @@ -55,6 +55,7 @@ public class Filter { public static final String JSON_OBJECT = "JSON_OBJECT"; public static final int THOUSAND = 1000; public static final String IS_NOT_NULL_TRIMMED = "IS NOT NULL"; + public static final String DIAMOND_EQUALS = "<=>"; public static String TEXT = "TEXT"; public static String OPTIONS = "OPTIONS"; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitter.java index d7335a9f7..9cb269166 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitter.java @@ -25,7 +25,7 @@ public String getInnerProperty() { return getFieldWithAlias()[1]; } - private String[] getFieldWithAlias() { + protected String[] getFieldWithAlias() { return splittedFilter[0].trim().split(ElasticSearchUtil.DOT_SEPARATOR); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitter.java new file mode 100644 index 000000000..4cb5e70b0 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitter.java @@ -0,0 +1,19 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import org.broadinstitute.dsm.model.Filter; + +public class DiamondEqualsSplitter extends BaseSplitter { + + @Override + protected String[] getFieldWithAlias() { + String[] fieldWithAlias = super.getFieldWithAlias(); + String alias = fieldWithAlias[0].split(" ")[1]; + String innerProperty = fieldWithAlias[1]; + return new String[] {alias, innerProperty}; + } + + @Override + public String[] split() { + return filter.split(Filter.DIAMOND_EQUALS); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java index 39e58359c..0f09e1dd4 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java @@ -7,7 +7,8 @@ public enum Operator { EQUALS(Filter.EQUALS_TRIMMED), GREATER_THAN_EQUALS(Filter.LARGER_EQUALS_TRIMMED), LESS_THAN_EQUALS(Filter.SMALLER_EQUALS_TRIMMED), - IS_NOT_NULL(Filter.IS_NOT_NULL_TRIMMED); + IS_NOT_NULL(Filter.IS_NOT_NULL_TRIMMED), + DIAMOND_EQUALS(Filter.DIAMOND_EQUALS); private String value; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java index c66c1eb27..9a913a16c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java @@ -5,6 +5,9 @@ public class SplitterFactory { public static BaseSplitter createSplitter(Operator operator) { BaseSplitter splitter; switch (operator) { + case DIAMOND_EQUALS: + splitter = new DiamondEqualsSplitter(); + break; case EQUALS: splitter = new EqualsSplitter(); break; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitterTest.java new file mode 100644 index 000000000..a5b669e15 --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitterTest.java @@ -0,0 +1,29 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import static org.junit.Assert.*; + +import org.broadinstitute.dsm.statics.ESObjectConstants; +import org.junit.Test; + +public class BaseSplitterTest { + + @Test + public void getAlias() { + DiamondEqualsSplitter diamondEqualsSplitter = getDiamondEqualsSplitter(); + assertEquals("m", diamondEqualsSplitter.getAlias()); + } + + @Test + public void getInnerProperty() { + DiamondEqualsSplitter diamondEqualsSplitter = getDiamondEqualsSplitter(); + assertEquals("international", diamondEqualsSplitter.getInnerProperty()); + } + + private DiamondEqualsSplitter getDiamondEqualsSplitter() { + String not = "NOT m.international <=> 1"; + DiamondEqualsSplitter diamondEqualsSplitter = new DiamondEqualsSplitter(); + diamondEqualsSplitter.setFilter(not); + return diamondEqualsSplitter; + } + +} \ No newline at end of file From f42d14fb98fa60db680758d97d9d0e1abd5e383b Mon Sep 17 00:00:00 2001 From: Nika Date: Fri, 17 Dec 2021 18:07:59 +0400 Subject: [PATCH 238/405] DDP-7121 | finish implementation of DiamondEqualsSplitter --- .../model/elastic/export/parse/BaseParser.java | 4 ++-- .../elastic/filter/DiamondEqualsSplitter.java | 17 ++++++++++++++++- .../dsm/model/elastic/filter/Operator.java | 3 ++- ...Test.java => DiamondEqualsSplitterTest.java} | 14 +++++++++----- .../model/elastic/filter/FilterParserTest.java | 2 ++ 5 files changed, 31 insertions(+), 9 deletions(-) rename src/test/java/org/broadinstitute/dsm/model/elastic/filter/{BaseSplitterTest.java => DiamondEqualsSplitterTest.java} (77%) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java index 31dc84ab4..e3cbd0514 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java @@ -68,9 +68,9 @@ protected boolean isBoolean(String value) { } public String convertBoolean(String value) { - if ("'1'".equals(value)) { + if ("'1'".equals(value) || "NOT'0'".equals(value)) { return "true"; - } else if ("'0'".equals(value)){ + } else if ("'0'".equals(value) || "NOT'1'".equals(value)) { return "false"; } else { return value; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitter.java index 4cb5e70b0..91e53ac36 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitter.java @@ -7,11 +7,26 @@ public class DiamondEqualsSplitter extends BaseSplitter { @Override protected String[] getFieldWithAlias() { String[] fieldWithAlias = super.getFieldWithAlias(); - String alias = fieldWithAlias[0].split(" ")[1]; + String alias = splitFieldWithAliasBySpace(fieldWithAlias)[1]; String innerProperty = fieldWithAlias[1]; return new String[] {alias, innerProperty}; } + private String[] splitFieldWithAliasBySpace(String[] fieldWithAlias) { + return fieldWithAlias[0].split(" "); + } + + @Override + public String getValue() { + String value = "'" + super.getValue() + "'"; + try { + String not = splitFieldWithAliasBySpace(super.getFieldWithAlias())[0]; + return not + value; + } catch (Exception e) { + return value; + } + } + @Override public String[] split() { return filter.split(Filter.DIAMOND_EQUALS); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java index 0f09e1dd4..901086a37 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java @@ -8,7 +8,8 @@ public enum Operator { GREATER_THAN_EQUALS(Filter.LARGER_EQUALS_TRIMMED), LESS_THAN_EQUALS(Filter.SMALLER_EQUALS_TRIMMED), IS_NOT_NULL(Filter.IS_NOT_NULL_TRIMMED), - DIAMOND_EQUALS(Filter.DIAMOND_EQUALS); + DIAMOND_EQUALS(Filter.DIAMOND_EQUALS), + DIAMOND_EQUALS(Filter.MUL); private String value; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitterTest.java similarity index 77% rename from src/test/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitterTest.java rename to src/test/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitterTest.java index a5b669e15..0c04f2f57 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitterTest.java @@ -1,11 +1,10 @@ package org.broadinstitute.dsm.model.elastic.filter; -import static org.junit.Assert.*; - -import org.broadinstitute.dsm.statics.ESObjectConstants; import org.junit.Test; -public class BaseSplitterTest { +import static org.junit.Assert.*; + +public class DiamondEqualsSplitterTest { @Test public void getAlias() { @@ -13,6 +12,12 @@ public void getAlias() { assertEquals("m", diamondEqualsSplitter.getAlias()); } + @Test + public void getValue() { + DiamondEqualsSplitter diamondEqualsSplitter = getDiamondEqualsSplitter(); + assertEquals("'1'", diamondEqualsSplitter.getValue()); + } + @Test public void getInnerProperty() { DiamondEqualsSplitter diamondEqualsSplitter = getDiamondEqualsSplitter(); @@ -25,5 +30,4 @@ private DiamondEqualsSplitter getDiamondEqualsSplitter() { diamondEqualsSplitter.setFilter(not); return diamondEqualsSplitter; } - } \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/FilterParserTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/FilterParserTest.java index be40ef20d..c1be91a1d 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/FilterParserTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/FilterParserTest.java @@ -10,12 +10,14 @@ public class FilterParserTest { public void parse() { String trueValue = "'1'"; + String falseValue = "NOT'1'"; String str = "'string'"; String number = "'5'"; String date = "'1999-05-22'"; BaseParser filterParser = new FilterParser(); Assert.assertEquals(true, filterParser.parse(trueValue)); + Assert.assertEquals(false, filterParser.parse(falseValue)); Assert.assertEquals("string", filterParser.parse(str)); Assert.assertEquals("5", filterParser.parse(number)); Assert.assertEquals("1999-05-22", filterParser.parse(date)); From 39167e42f8cb0261d9fbe30253396b6813990965 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 17 Dec 2021 18:35:43 +0400 Subject: [PATCH 239/405] DDP-7121 | add more special cases --- .../model/elastic/filter/BaseSplitter.java | 6 ++--- .../filter/MultipleOptionsSplitter.java | 23 +++++++++++++++++++ .../dsm/model/elastic/filter/Operator.java | 4 +++- .../model/elastic/filter/SplitterFactory.java | 3 +++ 4 files changed, 32 insertions(+), 4 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitter.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitter.java index 9cb269166..175398d0b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitter.java @@ -10,11 +10,11 @@ public abstract class BaseSplitter { public abstract String[] split(); - public String getValue() { + public String[] getValue() { if (splittedFilter.length > 1) { - return splittedFilter[1].trim(); + return new String[]{splittedFilter[1].trim()}; } - return StringUtils.EMPTY; + return new String[]{StringUtils.EMPTY}; } public String getAlias() { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitter.java new file mode 100644 index 000000000..4004ad0f2 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitter.java @@ -0,0 +1,23 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import org.apache.commons.lang3.StringUtils; +import org.broadinstitute.dsm.model.Filter; + +public class MultipleOptionsSplitter extends BaseSplitter { + + + @Override + public String[] getValue() { + return super.getValue(); + } + + @Override + public String[] split() { + String[] multipleFilters = filter + .replace(Filter.OPEN_PARENTHESIS, StringUtils.EMPTY) + .replace(Filter.CLOSE_PARENTHESIS, StringUtils.EMPTY) + .trim() + .split(Filter.OR_TRIMMED); + return multipleFilters; + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java index 901086a37..f9a9634c1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java @@ -9,7 +9,7 @@ public enum Operator { LESS_THAN_EQUALS(Filter.SMALLER_EQUALS_TRIMMED), IS_NOT_NULL(Filter.IS_NOT_NULL_TRIMMED), DIAMOND_EQUALS(Filter.DIAMOND_EQUALS), - DIAMOND_EQUALS(Filter.MUL); + MULTIPLE_OPTIONS(Filter.OPEN_PARENTHESIS); private String value; @@ -27,6 +27,8 @@ public static Operator getOperator(String value) { public static Operator extract(String filter) { if (filter.contains(Filter.IS_NOT_NULL_TRIMMED)) return IS_NOT_NULL; + else if (filter.startsWith(Filter.OPEN_PARENTHESIS)) + return MULTIPLE_OPTIONS; String operator = filter.split(" ")[1]; return getOperator(operator); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java index 9a913a16c..18041a3a7 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java @@ -5,6 +5,9 @@ public class SplitterFactory { public static BaseSplitter createSplitter(Operator operator) { BaseSplitter splitter; switch (operator) { + case MULTIPLE_OPTIONS: + splitter = new MultipleOptionsSplitter(); + break; case DIAMOND_EQUALS: splitter = new DiamondEqualsSplitter(); break; From fbff9ac991bfb2af1c9ec405ebceb9efa0d3e728 Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 20 Dec 2021 10:46:09 +0400 Subject: [PATCH 240/405] DDP-7121 | overload parse() method --- .../dsm/model/elastic/export/parse/Parser.java | 4 ++++ .../model/elastic/filter/DiamondEqualsSplitter.java | 8 ++++---- .../dsm/model/elastic/filter/FilterParser.java | 8 ++++++++ .../elastic/filter/MultipleOptionsSplitter.java | 13 +++++++++++-- .../model/elastic/filter/QueryBuilderFactory.java | 6 +++--- .../dsm/model/elastic/filter/QueryPayload.java | 7 ++++--- 6 files changed, 34 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/Parser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/Parser.java index 66d3d1911..996b985a7 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/Parser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/Parser.java @@ -3,4 +3,8 @@ public interface Parser { Object parse(String value); + + default Object[] parse(String[] values) { + throw new UnsupportedOperationException(); + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitter.java index 91e53ac36..2ed8d8050 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitter.java @@ -17,13 +17,13 @@ private String[] splitFieldWithAliasBySpace(String[] fieldWithAlias) { } @Override - public String getValue() { - String value = "'" + super.getValue() + "'"; + public String[] getValue() { + String value = "'" + super.getValue()[0] + "'"; try { String not = splitFieldWithAliasBySpace(super.getFieldWithAlias())[0]; - return not + value; + return new String[] { not + value }; } catch (Exception e) { - return value; + return new String[] { value }; } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/FilterParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/FilterParser.java index 87a66e242..fadfb4c58 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/FilterParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/FilterParser.java @@ -12,6 +12,14 @@ public Object parse(String value) { return convertString(value); } + @Override + public Object parse(String[] values) { + if (isBoolean(values)) + return forBoolean(convertBoolean(values)); + else + return convertString(values); + } + @Override protected Object forNumeric(String value) { return value; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitter.java index 4004ad0f2..daae75e3b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitter.java @@ -3,12 +3,21 @@ import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.Filter; -public class MultipleOptionsSplitter extends BaseSplitter { +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +public class MultipleOptionsSplitter extends BaseSplitter { @Override public String[] getValue() { - return super.getValue(); + List values = new ArrayList<>(); + for (String fieldValuePair : splittedFilter) { + String value = fieldValuePair.split(Filter.EQUALS_TRIMMED)[1]; + values.add(value); + } + return (String[]) values.toArray(); + } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java index 7c0acef01..6b64db45e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java @@ -11,16 +11,16 @@ public static QueryBuilder buildQueryBuilder(Operator operator, QueryPayload pay switch (operator) { case LIKE: case EQUALS: - queryBuilder = new MatchQueryBuilder(payload.getFieldName(), payload.getValue()); + queryBuilder = new MatchQueryBuilder(payload.getFieldName(), payload.getValues()); break; case GREATER_THAN_EQUALS: RangeQueryBuilder greaterRangeQuery = new RangeQueryBuilder(payload.getFieldName()); - greaterRangeQuery.gte(payload.getValue()); + greaterRangeQuery.gte(payload.getValues()); queryBuilder = greaterRangeQuery; break; case LESS_THAN_EQUALS: RangeQueryBuilder lessRangeQuery = new RangeQueryBuilder(payload.getFieldName()); - lessRangeQuery.lte(payload.getValue()); + lessRangeQuery.lte(payload.getValues()); queryBuilder = lessRangeQuery; break; case IS_NOT_NULL: diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryPayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryPayload.java index cf84d7edd..acd519ce1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryPayload.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryPayload.java @@ -4,12 +4,13 @@ @Getter public class QueryPayload { + String fieldName; - Object value; + Object[] values; - public QueryPayload(String fieldName, Object value) { + public QueryPayload(String fieldName, Object[] value) { this.fieldName = fieldName; - this.value = value; + this.values = value; } public QueryPayload(String fieldName) { From 6d5a70e961df0b133ac814e99218b1374ec0f755 Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 20 Dec 2021 12:04:29 +0400 Subject: [PATCH 241/405] DDP-7121 | change list to array in MultipleOptionsSplitter --- .../filter/CollectionQueryBuilder.java | 4 +-- .../model/elastic/filter/FilterParser.java | 17 +++++++---- .../filter/MultipleOptionsSplitter.java | 11 ++++---- .../elastic/filter/QueryBuilderFactory.java | 26 ++++++++--------- .../filter/CollectionQueryBuilderTest.java | 24 ++++++++++++++++ .../filter/QueryBuilderFactoryTest.java | 28 +++++++++---------- 6 files changed, 70 insertions(+), 40 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java index b23cdcdb8..8be6a0475 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java @@ -40,8 +40,8 @@ private void buildUpNestedQuery(BoolQueryBuilder boolQueryBuilder, List String outerProperty = Util.TABLE_ALIAS_MAPPINGS.get(splitter.getAlias()).getPropertyName(); //medicalRecord String nestedPath = DSM_WITH_DOT + outerProperty; QueryPayload queryPayload = new QueryPayload(nestedPath + "." + splitter.getInnerProperty(), parser.parse(splitter.getValue())); - QueryBuilder queryBuilder = QueryBuilderFactory.buildQueryBuilder(operator, queryPayload); - filterStrategy.build(boolQueryBuilder, new NestedQueryBuilder(nestedPath, queryBuilder, ScoreMode.Avg)); + QueryBuilderFactory.buildQueryBuilder(operator, queryPayload, boolQueryBuilder, filterStrategy); +// filterStrategy.build(boolQueryBuilder, new NestedQueryBuilder(nestedPath, queryBuilder, ScoreMode.Avg)); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/FilterParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/FilterParser.java index fadfb4c58..366336a56 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/FilterParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/FilterParser.java @@ -2,6 +2,9 @@ import org.broadinstitute.dsm.model.elastic.export.parse.ValueParser; +import java.util.ArrayList; +import java.util.List; + public class FilterParser extends ValueParser { @Override @@ -13,11 +16,15 @@ public Object parse(String value) { } @Override - public Object parse(String[] values) { - if (isBoolean(values)) - return forBoolean(convertBoolean(values)); - else - return convertString(values); + public Object[] parse(String[] values) { + List parsedValues = new ArrayList<>(); + for (String value : values) { + if (isBoolean(value)) + parsedValues.add(forBoolean(convertBoolean(value))); + else + parsedValues.add(convertString(value)); + } + return parsedValues.toArray(); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitter.java index daae75e3b..aaa7aa875 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitter.java @@ -11,13 +11,12 @@ public class MultipleOptionsSplitter extends BaseSplitter { @Override public String[] getValue() { - List values = new ArrayList<>(); - for (String fieldValuePair : splittedFilter) { - String value = fieldValuePair.split(Filter.EQUALS_TRIMMED)[1]; - values.add(value); + String[] values = new String[splittedFilter.length]; + for (int i = 0; i < values.length; i++) { + String value = splittedFilter[i].split(Filter.EQUALS_TRIMMED)[1]; + values[i] = value; } - return (String[]) values.toArray(); - + return values; } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java index 6b64db45e..389794453 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java @@ -1,34 +1,34 @@ package org.broadinstitute.dsm.model.elastic.filter; -import org.elasticsearch.index.query.ExistsQueryBuilder; -import org.elasticsearch.index.query.MatchQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.index.query.*; public class QueryBuilderFactory { - public static QueryBuilder buildQueryBuilder(Operator operator, QueryPayload payload) { - QueryBuilder queryBuilder; + public static void buildQueryBuilder(Operator operator, QueryPayload payload, BoolQueryBuilder boolQueryBuilder, FilterStrategy filterStrategy) { switch (operator) { case LIKE: case EQUALS: - queryBuilder = new MatchQueryBuilder(payload.getFieldName(), payload.getValues()); + filterStrategy.build(boolQueryBuilder, new MatchQueryBuilder(payload.getFieldName(), payload.getValues()[0])); break; case GREATER_THAN_EQUALS: RangeQueryBuilder greaterRangeQuery = new RangeQueryBuilder(payload.getFieldName()); - greaterRangeQuery.gte(payload.getValues()); - queryBuilder = greaterRangeQuery; + greaterRangeQuery.gte(payload.getValues()[0]); + filterStrategy.build(boolQueryBuilder, greaterRangeQuery); break; case LESS_THAN_EQUALS: RangeQueryBuilder lessRangeQuery = new RangeQueryBuilder(payload.getFieldName()); - lessRangeQuery.lte(payload.getValues()); - queryBuilder = lessRangeQuery; + lessRangeQuery.lte(payload.getValues()[0]); + filterStrategy.build(boolQueryBuilder, lessRangeQuery); break; case IS_NOT_NULL: - queryBuilder = new ExistsQueryBuilder(payload.getFieldName()); + filterStrategy.build(boolQueryBuilder, new ExistsQueryBuilder(payload.getFieldName())); + break; + case MULTIPLE_OPTIONS: + Object[] values = payload.getValues(); + for (Object value : values) + filterStrategy.build(boolQueryBuilder, new MatchQueryBuilder(payload.getFieldName(), value)); break; default: throw new IllegalArgumentException("Unknown operator"); } - return queryBuilder; } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java index 8fc58f768..7e9d0f5e4 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java @@ -102,6 +102,30 @@ public void collectionBuildBoolean() { Assert.assertEquals(expected, actual); } + @Test + public void multipleOptionsQueryBuilder() { + + String filter = "AND ( oD.request = 'review' OR oD.request = 'no' OR oD.request = 'hold' OR oD.request = 'request' OR oD.request = 'unable To Obtain' OR oD.request = 'sent' OR oD.request = 'received' OR oD.request = 'returned' )"; + + collectionQueryBuilder.setFilter(filter); + AbstractQueryBuilder actual = collectionQueryBuilder.build(); + + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); + boolQueryBuilder.should(new MatchQueryBuilder("dsm.oncHistoryDetail.request", "review")); + boolQueryBuilder.should(new MatchQueryBuilder("dsm.oncHistoryDetail.request", "no")); + boolQueryBuilder.should(new MatchQueryBuilder("dsm.oncHistoryDetail.request", "hold")); + boolQueryBuilder.should(new MatchQueryBuilder("dsm.oncHistoryDetail.request", "request")); + boolQueryBuilder.should(new MatchQueryBuilder("dsm.oncHistoryDetail.request", "unable To Obtain")); + boolQueryBuilder.should(new MatchQueryBuilder("dsm.oncHistoryDetail.request", "sent")); + boolQueryBuilder.should(new MatchQueryBuilder("dsm.oncHistoryDetail.request", "received")); + boolQueryBuilder.should(new MatchQueryBuilder("dsm.oncHistoryDetail.request", "returned")); + + AbstractQueryBuilder expected = new BoolQueryBuilder().must(new NestedQueryBuilder("dsm.medicalRecord", + boolQueryBuilder, ScoreMode.Avg)); + + Assert.assertEquals(expected, actual); + } + } \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactoryTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactoryTest.java index d14dbb4a4..fd70291a2 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactoryTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactoryTest.java @@ -9,19 +9,19 @@ public class QueryBuilderFactoryTest { - - @Test - public void buildQueryBuilder() { - QueryPayload payload = new QueryPayload("medicalRecordId", 15); - Operator operator = Operator.EQUALS; - QueryBuilder queryBuilder = QueryBuilderFactory.buildQueryBuilder(operator, payload); - assertTrue(queryBuilder instanceof MatchQueryBuilder); - operator = Operator.LIKE; - queryBuilder = QueryBuilderFactory.buildQueryBuilder(operator, payload); - assertTrue(queryBuilder instanceof MatchQueryBuilder); - operator = Operator.GREATER_THAN_EQUALS; - queryBuilder = QueryBuilderFactory.buildQueryBuilder(operator, payload); - assertTrue(queryBuilder instanceof RangeQueryBuilder); - } +// +// @Test +// public void buildQueryBuilder() { +// QueryPayload payload = new QueryPayload("medicalRecordId", 15); +// Operator operator = Operator.EQUALS; +// QueryBuilder queryBuilder = QueryBuilderFactory.buildQueryBuilder(operator, payload, boolQueryBuilder, filterStrategy); +// assertTrue(queryBuilder instanceof MatchQueryBuilder); +// operator = Operator.LIKE; +// queryBuilder = QueryBuilderFactory.buildQueryBuilder(operator, payload, boolQueryBuilder, filterStrategy); +// assertTrue(queryBuilder instanceof MatchQueryBuilder); +// operator = Operator.GREATER_THAN_EQUALS; +// queryBuilder = QueryBuilderFactory.buildQueryBuilder(operator, payload, boolQueryBuilder, filterStrategy); +// assertTrue(queryBuilder instanceof RangeQueryBuilder); +// } } \ No newline at end of file From dec191daa2d79e007670cdadeb08d03e5334a1b6 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 20 Dec 2021 15:39:34 +0400 Subject: [PATCH 242/405] DDP-7121 | refactor querybuilder factory to fit multiple options as well --- .../filter/CollectionQueryBuilder.java | 8 +++--- .../filter/MultipleOptionsSplitter.java | 9 ++++++- .../elastic/filter/QueryBuilderFactory.java | 25 +++++++++++++------ .../model/elastic/filter/QueryPayload.java | 14 ++++++----- .../filter/CollectionQueryBuilderTest.java | 4 +-- 5 files changed, 39 insertions(+), 21 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java index 8be6a0475..0c962db97 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java @@ -39,9 +39,9 @@ private void buildUpNestedQuery(BoolQueryBuilder boolQueryBuilder, List splitter.setFilter(filterValue); String outerProperty = Util.TABLE_ALIAS_MAPPINGS.get(splitter.getAlias()).getPropertyName(); //medicalRecord String nestedPath = DSM_WITH_DOT + outerProperty; - QueryPayload queryPayload = new QueryPayload(nestedPath + "." + splitter.getInnerProperty(), parser.parse(splitter.getValue())); - QueryBuilderFactory.buildQueryBuilder(operator, queryPayload, boolQueryBuilder, filterStrategy); -// filterStrategy.build(boolQueryBuilder, new NestedQueryBuilder(nestedPath, queryBuilder, ScoreMode.Avg)); + QueryPayload queryPayload = new QueryPayload(nestedPath, splitter.getInnerProperty(), parser.parse(splitter.getValue())); + QueryBuilder queryBuilder = QueryBuilderFactory.buildQueryBuilder(operator, queryPayload); + filterStrategy.build(boolQueryBuilder, new NestedQueryBuilder(nestedPath, queryBuilder, ScoreMode.Avg)); } } @@ -51,7 +51,7 @@ protected Map> parseFiltersByLogicalOperators() { String[] andSeparated = filter.split(Filter.AND_TRIMMED); for (String eachFilter : andSeparated) { String cleanedEachFilter = eachFilter.trim(); - if (cleanedEachFilter.contains(Filter.OR_TRIMMED)) { + if (cleanedEachFilter.contains(Filter.OR_TRIMMED) && !cleanedEachFilter.startsWith(Filter.OPEN_PARENTHESIS) && !cleanedEachFilter.endsWith(Filter.CLOSE_PARENTHESIS)) { String[] orSeparated = cleanedEachFilter.split(Filter.OR_TRIMMED); filterByLogicalOperators.get(Filter.AND_TRIMMED).add(orSeparated[0].trim()); Arrays.stream(orSeparated) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitter.java index aaa7aa875..1274350c8 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitter.java @@ -13,12 +13,19 @@ public class MultipleOptionsSplitter extends BaseSplitter { public String[] getValue() { String[] values = new String[splittedFilter.length]; for (int i = 0; i < values.length; i++) { - String value = splittedFilter[i].split(Filter.EQUALS_TRIMMED)[1]; + String value = splittedFilter[i].split(Filter.EQUALS_TRIMMED)[1].trim(); values[i] = value; } return values; } + @Override + public String getInnerProperty() { + String propertyWithValue = super.getInnerProperty(); + String innerProperty = propertyWithValue.split(Filter.EQUALS_TRIMMED)[0].trim(); + return innerProperty; + } + @Override public String[] split() { String[] multipleFilters = filter diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java index 389794453..46155a645 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java @@ -1,34 +1,43 @@ package org.broadinstitute.dsm.model.elastic.filter; -import org.elasticsearch.index.query.*; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.ExistsQueryBuilder; +import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.RangeQueryBuilder; public class QueryBuilderFactory { - public static void buildQueryBuilder(Operator operator, QueryPayload payload, BoolQueryBuilder boolQueryBuilder, FilterStrategy filterStrategy) { + public static QueryBuilder buildQueryBuilder(Operator operator, QueryPayload payload) { + QueryBuilder qb; switch (operator) { case LIKE: case EQUALS: - filterStrategy.build(boolQueryBuilder, new MatchQueryBuilder(payload.getFieldName(), payload.getValues()[0])); + qb = new MatchQueryBuilder(payload.getFieldName(), payload.getValues()[0]); break; case GREATER_THAN_EQUALS: RangeQueryBuilder greaterRangeQuery = new RangeQueryBuilder(payload.getFieldName()); greaterRangeQuery.gte(payload.getValues()[0]); - filterStrategy.build(boolQueryBuilder, greaterRangeQuery); + qb = greaterRangeQuery; break; case LESS_THAN_EQUALS: RangeQueryBuilder lessRangeQuery = new RangeQueryBuilder(payload.getFieldName()); lessRangeQuery.lte(payload.getValues()[0]); - filterStrategy.build(boolQueryBuilder, lessRangeQuery); + qb = lessRangeQuery; break; case IS_NOT_NULL: - filterStrategy.build(boolQueryBuilder, new ExistsQueryBuilder(payload.getFieldName())); + qb = new ExistsQueryBuilder(payload.getFieldName()); break; case MULTIPLE_OPTIONS: + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); Object[] values = payload.getValues(); - for (Object value : values) - filterStrategy.build(boolQueryBuilder, new MatchQueryBuilder(payload.getFieldName(), value)); + for (Object value : values) { + boolQueryBuilder.should(new MatchQueryBuilder(payload.getFieldName(), value)); + } + qb = boolQueryBuilder; break; default: throw new IllegalArgumentException("Unknown operator"); } + return qb; } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryPayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryPayload.java index acd519ce1..f67eaa9f0 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryPayload.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryPayload.java @@ -5,15 +5,17 @@ @Getter public class QueryPayload { - String fieldName; + private String property; + private String path; Object[] values; - public QueryPayload(String fieldName, Object[] value) { - this.fieldName = fieldName; - this.values = value; + public QueryPayload(String path, String property, Object[] values) { + this.path = path; + this.property = property; + this.values = values; } - public QueryPayload(String fieldName) { - this.fieldName = fieldName; + public String getFieldName() { + return path + "." + property; } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java index 7e9d0f5e4..cbb5e7f30 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java @@ -97,7 +97,7 @@ public void collectionBuildBoolean() { AbstractQueryBuilder actual = collectionQueryBuilder.build(); AbstractQueryBuilder expected = new BoolQueryBuilder().must(new NestedQueryBuilder("dsm.medicalRecord", - new MatchQueryBuilder("dsm.medicalRecord.followUp", "1"), ScoreMode.Avg)); + new MatchQueryBuilder("dsm.medicalRecord.followUp", true), ScoreMode.Avg)); Assert.assertEquals(expected, actual); } @@ -120,7 +120,7 @@ public void multipleOptionsQueryBuilder() { boolQueryBuilder.should(new MatchQueryBuilder("dsm.oncHistoryDetail.request", "received")); boolQueryBuilder.should(new MatchQueryBuilder("dsm.oncHistoryDetail.request", "returned")); - AbstractQueryBuilder expected = new BoolQueryBuilder().must(new NestedQueryBuilder("dsm.medicalRecord", + AbstractQueryBuilder expected = new BoolQueryBuilder().must(new NestedQueryBuilder("dsm.oncHistoryDetail", boolQueryBuilder, ScoreMode.Avg)); Assert.assertEquals(expected, actual); From d5987bab7cc20ebe0dcfbf7face01723eb588342 Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 20 Dec 2021 16:51:34 +0400 Subject: [PATCH 243/405] DDP-7121 | add DateSplitter for date filtering --- .../org/broadinstitute/dsm/model/Filter.java | 1 + .../model/elastic/filter/BaseSplitter.java | 5 ++-- .../model/elastic/filter/DateSplitter.java | 22 +++++++++++++++ .../elastic/filter/DiamondEqualsSplitter.java | 3 +- .../dsm/model/elastic/filter/Operator.java | 5 +++- .../model/elastic/filter/SplitterFactory.java | 3 ++ .../elastic/filter/DateSplitterTest.java | 20 +++++++++++++ .../filter/DiamondEqualsSplitterTest.java | 6 ++-- .../filter/QueryBuilderFactoryTest.java | 28 +++++++++---------- 9 files changed, 72 insertions(+), 21 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitter.java create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitterTest.java diff --git a/src/main/java/org/broadinstitute/dsm/model/Filter.java b/src/main/java/org/broadinstitute/dsm/model/Filter.java index be0dfde98..0f31d7b06 100644 --- a/src/main/java/org/broadinstitute/dsm/model/Filter.java +++ b/src/main/java/org/broadinstitute/dsm/model/Filter.java @@ -56,6 +56,7 @@ public class Filter { public static final int THOUSAND = 1000; public static final String IS_NOT_NULL_TRIMMED = "IS NOT NULL"; public static final String DIAMOND_EQUALS = "<=>"; + public static final String DATE_FORMAT = "STR_TO_DATE"; public static String TEXT = "TEXT"; public static String OPTIONS = "OPTIONS"; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitter.java index 175398d0b..a979fe77c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitter.java @@ -1,6 +1,7 @@ package org.broadinstitute.dsm.model.elastic.filter; import org.apache.commons.lang3.StringUtils; +import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.util.ElasticSearchUtil; public abstract class BaseSplitter { @@ -12,7 +13,7 @@ public abstract class BaseSplitter { public String[] getValue() { if (splittedFilter.length > 1) { - return new String[]{splittedFilter[1].trim()}; + return new String[]{splittedFilter[1].trim() }; } return new String[]{StringUtils.EMPTY}; } @@ -22,7 +23,7 @@ public String getAlias() { } public String getInnerProperty() { - return getFieldWithAlias()[1]; + return Util.underscoresToCamelCase(getFieldWithAlias()[1]); } protected String[] getFieldWithAlias() { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitter.java new file mode 100644 index 000000000..2bcba02e8 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitter.java @@ -0,0 +1,22 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import org.apache.commons.lang3.StringUtils; +import org.broadinstitute.dsm.model.Filter; + +public class DateSplitter extends BaseSplitter { + + + @Override + public String[] split() { + // STR_TO_DATE(m.fax_sent,'%Y-%m-%d') = STR_TO_DATE('2021-12-17','%Y-%m-%d') + String[] splittedFilter = new String[2]; + String[] dateFieldWithValue = filter.split(Filter.EQUALS_TRIMMED); + for (int i = 0; i < dateFieldWithValue.length; i++) { + splittedFilter[i] = dateFieldWithValue[i].split(Filter.DATE_FORMAT)[1] + .replace(Filter.OPEN_PARENTHESIS, StringUtils.EMPTY) + .replace(Filter.CLOSE_PARENTHESIS, StringUtils.EMPTY) + .split(",")[0]; + } + return splittedFilter; + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitter.java index 2ed8d8050..f73ce990e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitter.java @@ -1,6 +1,7 @@ package org.broadinstitute.dsm.model.elastic.filter; import org.broadinstitute.dsm.model.Filter; +import org.broadinstitute.dsm.model.elastic.Util; public class DiamondEqualsSplitter extends BaseSplitter { @@ -8,7 +9,7 @@ public class DiamondEqualsSplitter extends BaseSplitter { protected String[] getFieldWithAlias() { String[] fieldWithAlias = super.getFieldWithAlias(); String alias = splitFieldWithAliasBySpace(fieldWithAlias)[1]; - String innerProperty = fieldWithAlias[1]; + String innerProperty = Util.underscoresToCamelCase(fieldWithAlias[1]); return new String[] {alias, innerProperty}; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java index f9a9634c1..db268db1e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java @@ -9,7 +9,8 @@ public enum Operator { LESS_THAN_EQUALS(Filter.SMALLER_EQUALS_TRIMMED), IS_NOT_NULL(Filter.IS_NOT_NULL_TRIMMED), DIAMOND_EQUALS(Filter.DIAMOND_EQUALS), - MULTIPLE_OPTIONS(Filter.OPEN_PARENTHESIS); + MULTIPLE_OPTIONS(Filter.OPEN_PARENTHESIS), + DATE(Filter.DATE_FORMAT); private String value; @@ -29,6 +30,8 @@ public static Operator extract(String filter) { return IS_NOT_NULL; else if (filter.startsWith(Filter.OPEN_PARENTHESIS)) return MULTIPLE_OPTIONS; + else if (filter.startsWith(Filter.DATE_FORMAT)) + return DATE; String operator = filter.split(" ")[1]; return getOperator(operator); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java index 18041a3a7..3cc68b90d 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java @@ -23,6 +23,9 @@ public static BaseSplitter createSplitter(Operator operator) { case LESS_THAN_EQUALS: splitter = new LessThanEqualsSplitter(); break; + case DATE: + splitter = new DateSplitter(); + break; case IS_NOT_NULL: splitter = new IsNotNullSplitter(); break; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitterTest.java new file mode 100644 index 000000000..e36c2c14a --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitterTest.java @@ -0,0 +1,20 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import org.junit.Assert; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class DateSplitterTest { + + @Test + public void split() { + + BaseSplitter dateSplitter = SplitterFactory.createSplitter(Operator.DATE); + dateSplitter.setFilter("STR_TO_DATE(m.fax_sent,'%Y-%m-%d') = STR_TO_DATE('2021-12-17','%Y-%m-%d')"); + + Assert.assertEquals("m", dateSplitter.getAlias()); + Assert.assertEquals("faxSent", dateSplitter.getInnerProperty()); + Assert.assertEquals("'2021-12-17'", dateSplitter.getValue()[0]); + } +} \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitterTest.java index 0c04f2f57..39d71eeb0 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitterTest.java @@ -15,17 +15,17 @@ public void getAlias() { @Test public void getValue() { DiamondEqualsSplitter diamondEqualsSplitter = getDiamondEqualsSplitter(); - assertEquals("'1'", diamondEqualsSplitter.getValue()); + assertEquals("'1'", diamondEqualsSplitter.getValue()[0]); } @Test public void getInnerProperty() { DiamondEqualsSplitter diamondEqualsSplitter = getDiamondEqualsSplitter(); - assertEquals("international", diamondEqualsSplitter.getInnerProperty()); + assertEquals("faxSent", diamondEqualsSplitter.getInnerProperty()); } private DiamondEqualsSplitter getDiamondEqualsSplitter() { - String not = "NOT m.international <=> 1"; + String not = "NOT m.fax_sent <=> 1"; DiamondEqualsSplitter diamondEqualsSplitter = new DiamondEqualsSplitter(); diamondEqualsSplitter.setFilter(not); return diamondEqualsSplitter; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactoryTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactoryTest.java index fd70291a2..40ad0d553 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactoryTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactoryTest.java @@ -9,19 +9,19 @@ public class QueryBuilderFactoryTest { -// -// @Test -// public void buildQueryBuilder() { -// QueryPayload payload = new QueryPayload("medicalRecordId", 15); -// Operator operator = Operator.EQUALS; -// QueryBuilder queryBuilder = QueryBuilderFactory.buildQueryBuilder(operator, payload, boolQueryBuilder, filterStrategy); -// assertTrue(queryBuilder instanceof MatchQueryBuilder); -// operator = Operator.LIKE; -// queryBuilder = QueryBuilderFactory.buildQueryBuilder(operator, payload, boolQueryBuilder, filterStrategy); -// assertTrue(queryBuilder instanceof MatchQueryBuilder); -// operator = Operator.GREATER_THAN_EQUALS; -// queryBuilder = QueryBuilderFactory.buildQueryBuilder(operator, payload, boolQueryBuilder, filterStrategy); -// assertTrue(queryBuilder instanceof RangeQueryBuilder); -// } + + @Test + public void buildQueryBuilder() { + QueryPayload payload = new QueryPayload("dsm.medicalRecord", "medicalRecordId", new Integer[] {10}); + Operator operator = Operator.EQUALS; + QueryBuilder queryBuilder = QueryBuilderFactory.buildQueryBuilder(operator, payload); + assertTrue(queryBuilder instanceof MatchQueryBuilder); + operator = Operator.LIKE; + queryBuilder = QueryBuilderFactory.buildQueryBuilder(operator, payload); + assertTrue(queryBuilder instanceof MatchQueryBuilder); + operator = Operator.GREATER_THAN_EQUALS; + queryBuilder = QueryBuilderFactory.buildQueryBuilder(operator, payload); + assertTrue(queryBuilder instanceof RangeQueryBuilder); + } } \ No newline at end of file From 501f3542ea652ed59990702b6652d5fcb432f8ab Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 20 Dec 2021 17:23:31 +0400 Subject: [PATCH 244/405] DDP-7121 | test fixes, add date case for query builder factory --- .../elastic/filter/MultipleOptionsSplitter.java | 7 ++----- .../elastic/filter/QueryBuilderFactory.java | 3 +++ .../filter/DiamondEqualsSplitterTest.java | 2 +- .../filter/MultipleOptionsSplitterTest.java | 17 +++++++++++++++++ 4 files changed, 23 insertions(+), 6 deletions(-) create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitterTest.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitter.java index 1274350c8..03799f65c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitter.java @@ -2,10 +2,7 @@ import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.Filter; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; +import org.broadinstitute.dsm.model.elastic.Util; public class MultipleOptionsSplitter extends BaseSplitter { @@ -23,7 +20,7 @@ public String[] getValue() { public String getInnerProperty() { String propertyWithValue = super.getInnerProperty(); String innerProperty = propertyWithValue.split(Filter.EQUALS_TRIMMED)[0].trim(); - return innerProperty; + return Util.underscoresToCamelCase(innerProperty); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java index 46155a645..fa095a283 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java @@ -27,6 +27,9 @@ public static QueryBuilder buildQueryBuilder(Operator operator, QueryPayload pay case IS_NOT_NULL: qb = new ExistsQueryBuilder(payload.getFieldName()); break; + case DATE: + qb = new MatchQueryBuilder(payload.getFieldName(), payload.getValues()[0]); + break; case MULTIPLE_OPTIONS: BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); Object[] values = payload.getValues(); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitterTest.java index 39d71eeb0..264629df0 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitterTest.java @@ -15,7 +15,7 @@ public void getAlias() { @Test public void getValue() { DiamondEqualsSplitter diamondEqualsSplitter = getDiamondEqualsSplitter(); - assertEquals("'1'", diamondEqualsSplitter.getValue()[0]); + assertEquals("NOT'1'", diamondEqualsSplitter.getValue()[0]); } @Test diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitterTest.java new file mode 100644 index 000000000..a02fe2b4e --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitterTest.java @@ -0,0 +1,17 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import static org.junit.Assert.*; + +import org.junit.Test; + +public class MultipleOptionsSplitterTest { + + @Test + public void getInnerProperty() { + String filter = "( oD.fax_sent = 'review' OR oD.fax_sent = 'no' OR oD.fax_sent = 'hold' OR oD.fax_sent = 'request' OR oD.fax_sent = " + + "'unable To Obtain' OR oD.fax_sent = 'sent' OR oD.fax_sent = 'received' OR oD.fax_sent = 'returned' )"; + BaseSplitter multipleSplitter = SplitterFactory.createSplitter(Operator.MULTIPLE_OPTIONS); + multipleSplitter.setFilter(filter); + assertEquals("faxSent", multipleSplitter.getInnerProperty()); + } +} \ No newline at end of file From e368a8ff36d15728b677ac2c833a6b53a37e5297 Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 20 Dec 2021 18:56:18 +0400 Subject: [PATCH 245/405] DDP-7121 | take care of Date filtering with =, >= and <= logical operators --- .../org/broadinstitute/dsm/model/Filter.java | 2 ++ .../elastic/filter/DateGreaterSplitter.java | 16 ++++++++++++++++ .../model/elastic/filter/DateLowerSplitter.java | 16 ++++++++++++++++ .../dsm/model/elastic/filter/Operator.java | 8 +++++++- .../elastic/filter/QueryBuilderFactory.java | 14 +++++++++++--- .../model/elastic/filter/SplitterFactory.java | 6 ++++++ .../filter/CollectionQueryBuilderTest.java | 16 ++++++++++++++++ .../elastic/filter/DateGreaterSplitterTest.java | 17 +++++++++++++++++ .../elastic/filter/DateLowerSplitterTest.java | 17 +++++++++++++++++ 9 files changed, 108 insertions(+), 4 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/DateGreaterSplitter.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/DateLowerSplitter.java create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateGreaterSplitterTest.java create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateLowerSplitterTest.java diff --git a/src/main/java/org/broadinstitute/dsm/model/Filter.java b/src/main/java/org/broadinstitute/dsm/model/Filter.java index 0f31d7b06..1085988d7 100644 --- a/src/main/java/org/broadinstitute/dsm/model/Filter.java +++ b/src/main/java/org/broadinstitute/dsm/model/Filter.java @@ -57,6 +57,8 @@ public class Filter { public static final String IS_NOT_NULL_TRIMMED = "IS NOT NULL"; public static final String DIAMOND_EQUALS = "<=>"; public static final String DATE_FORMAT = "STR_TO_DATE"; + public static final String DATE_GREATER = LARGER_EQUALS + DATE_FORMAT; + public static final String DATE_LESS = SMALLER_EQUALS + DATE_FORMAT; public static String TEXT = "TEXT"; public static String OPTIONS = "OPTIONS"; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DateGreaterSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DateGreaterSplitter.java new file mode 100644 index 000000000..63669280d --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DateGreaterSplitter.java @@ -0,0 +1,16 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import org.apache.commons.lang3.StringUtils; +import org.broadinstitute.dsm.model.Filter; + +public class DateGreaterSplitter extends GreaterThanEqualsSplitter { + + @Override + public String[] getValue() { + return new String[]{splittedFilter[1].split(Filter.DATE_FORMAT)[1] + .replace(Filter.OPEN_PARENTHESIS, StringUtils.EMPTY) + .replace(Filter.CLOSE_PARENTHESIS, StringUtils.EMPTY) + .split(",")[0]}; + + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DateLowerSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DateLowerSplitter.java new file mode 100644 index 000000000..ac08402bf --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DateLowerSplitter.java @@ -0,0 +1,16 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import org.apache.commons.lang3.StringUtils; +import org.broadinstitute.dsm.model.Filter; + +public class DateLowerSplitter extends LessThanEqualsSplitter { + + @Override + public String[] getValue() { + // STR_TO_DATE('2012-01-01', %yyyy-%MM-%dd) + return new String[]{splittedFilter[1].split(Filter.DATE_FORMAT)[1] + .replace(Filter.OPEN_PARENTHESIS, StringUtils.EMPTY) + .replace(Filter.CLOSE_PARENTHESIS, StringUtils.EMPTY) + .split(",")[0]}; + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java index db268db1e..89f18c6a5 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java @@ -10,7 +10,9 @@ public enum Operator { IS_NOT_NULL(Filter.IS_NOT_NULL_TRIMMED), DIAMOND_EQUALS(Filter.DIAMOND_EQUALS), MULTIPLE_OPTIONS(Filter.OPEN_PARENTHESIS), - DATE(Filter.DATE_FORMAT); + DATE(Filter.DATE_FORMAT), + DATE_GREATER(Filter.DATE_GREATER), + DATE_LESS(Filter.DATE_LESS); private String value; @@ -32,6 +34,10 @@ else if (filter.startsWith(Filter.OPEN_PARENTHESIS)) return MULTIPLE_OPTIONS; else if (filter.startsWith(Filter.DATE_FORMAT)) return DATE; + else if (filter.contains(Filter.DATE_GREATER)) + return DATE_GREATER; + else if (filter.contains(Filter.DATE_LESS)) + return DATE_LESS; String operator = filter.split(" ")[1]; return getOperator(operator); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java index fa095a283..8336e288b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java @@ -12,6 +12,7 @@ public static QueryBuilder buildQueryBuilder(Operator operator, QueryPayload pay switch (operator) { case LIKE: case EQUALS: + case DATE: qb = new MatchQueryBuilder(payload.getFieldName(), payload.getValues()[0]); break; case GREATER_THAN_EQUALS: @@ -24,12 +25,19 @@ public static QueryBuilder buildQueryBuilder(Operator operator, QueryPayload pay lessRangeQuery.lte(payload.getValues()[0]); qb = lessRangeQuery; break; + case DATE_GREATER: + RangeQueryBuilder dateGreaterQuery = new RangeQueryBuilder(payload.getFieldName()); + dateGreaterQuery.gte(payload.getValues()[0]); + qb = dateGreaterQuery; + break; + case DATE_LESS: + RangeQueryBuilder dateLessQuery = new RangeQueryBuilder(payload.getFieldName()); + dateLessQuery.lte(payload.getValues()[0]); + qb = dateLessQuery; + break; case IS_NOT_NULL: qb = new ExistsQueryBuilder(payload.getFieldName()); break; - case DATE: - qb = new MatchQueryBuilder(payload.getFieldName(), payload.getValues()[0]); - break; case MULTIPLE_OPTIONS: BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); Object[] values = payload.getValues(); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java index 3cc68b90d..6e6cc87d7 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java @@ -26,6 +26,12 @@ public static BaseSplitter createSplitter(Operator operator) { case DATE: splitter = new DateSplitter(); break; + case DATE_GREATER: + splitter = new DateGreaterSplitter(); + break; + case DATE_LESS: + splitter = new DateLowerSplitter(); + break; case IS_NOT_NULL: splitter = new IsNotNullSplitter(); break; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java index cbb5e7f30..e60a4e1e0 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java @@ -126,6 +126,22 @@ public void multipleOptionsQueryBuilder() { Assert.assertEquals(expected, actual); } + @Test + public void dateGreaterBuild() { + + String filter = "AND m.received >= STR_TO_DATE('2012-01-01', %yyyy-%MM-%dd) AND m.received <= STR_TO_DATE('2015-01-01', %yyyy-%MM-%dd)"; + + collectionQueryBuilder.setFilter(filter); + AbstractQueryBuilder actual = collectionQueryBuilder.build(); + + AbstractQueryBuilder expected = new BoolQueryBuilder().must(new NestedQueryBuilder("dsm.medicalRecord", + new RangeQueryBuilder("dsm.medicalRecord.received").gte("2012-01-01"), ScoreMode.Avg)) + .must(new NestedQueryBuilder("dsm.medicalRecord", new RangeQueryBuilder("dsm.medicalRecord.received").lte("2015-01-01"), + ScoreMode.Avg)); + + Assert.assertEquals(expected, actual); + } + } \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateGreaterSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateGreaterSplitterTest.java new file mode 100644 index 000000000..21061c2ce --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateGreaterSplitterTest.java @@ -0,0 +1,17 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import org.junit.Assert; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class DateGreaterSplitterTest { + + @Test + public void getValue() { + BaseSplitter splitter = SplitterFactory.createSplitter(Operator.DATE_GREATER); + String filter = "AND m.mr_received >= STR_TO_DATE('2000-01-01','%Y-%m-%d')"; + splitter.setFilter(filter); + Assert.assertEquals("'2000-01-01'", splitter.getValue()[0]); + } +} \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateLowerSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateLowerSplitterTest.java new file mode 100644 index 000000000..78e0a6279 --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateLowerSplitterTest.java @@ -0,0 +1,17 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import org.junit.Assert; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class DateLowerSplitterTest { + + @Test + public void getValue() { + BaseSplitter splitter = SplitterFactory.createSplitter(Operator.DATE_LESS); + String filter = "AND m.mr_received <= STR_TO_DATE('2000-01-01','%Y-%m-%d')"; + splitter.setFilter(filter); + Assert.assertEquals("'2000-01-01'", splitter.getValue()[0]); + } +} \ No newline at end of file From ed2fda3241d9727fc1fb17b7d7809868493915ae Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 21 Dec 2021 10:50:15 +0400 Subject: [PATCH 246/405] DDP-7121 | add json extract splitter for dynamic fields --- .../dsm/model/elastic/filter/JsonExtractSplitter.java | 7 +++++++ .../broadinstitute/dsm/model/elastic/filter/Operator.java | 5 ++++- .../dsm/model/elastic/filter/SplitterFactory.java | 3 +++ 3 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitter.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitter.java new file mode 100644 index 000000000..7272acdaf --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitter.java @@ -0,0 +1,7 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +public class JsonExtractSplitter extends EqualsSplitter { + + //JSON_EXTRACT ( m.additional_values_json , '$.seeingIfBugExists' ) = 'true' + +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java index 89f18c6a5..7fbec1716 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java @@ -12,7 +12,8 @@ public enum Operator { MULTIPLE_OPTIONS(Filter.OPEN_PARENTHESIS), DATE(Filter.DATE_FORMAT), DATE_GREATER(Filter.DATE_GREATER), - DATE_LESS(Filter.DATE_LESS); + DATE_LESS(Filter.DATE_LESS), + JSON_EXTRACT(Filter.JSON_EXTRACT); private String value; @@ -38,6 +39,8 @@ else if (filter.contains(Filter.DATE_GREATER)) return DATE_GREATER; else if (filter.contains(Filter.DATE_LESS)) return DATE_LESS; + else if (filter.contains(Filter.JSON_EXTRACT)) + return JSON_EXTRACT; String operator = filter.split(" ")[1]; return getOperator(operator); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java index 6e6cc87d7..fc38fdc46 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java @@ -35,6 +35,9 @@ public static BaseSplitter createSplitter(Operator operator) { case IS_NOT_NULL: splitter = new IsNotNullSplitter(); break; + case JSON_EXTRACT: + splitter = new JsonExtractSplitter(); + break; default: throw new IllegalArgumentException("Unknown operator"); } From 023e28e6ee3fd43dabac807fb56735caea6f3e32 Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 21 Dec 2021 11:25:34 +0400 Subject: [PATCH 247/405] DDP-7121 | add JsonExtractSplitter, make unit tests pass --- .../elastic/filter/JsonExtractSplitter.java | 25 +++++++++++++++- .../filter/JsonExtractSplitterTest.java | 30 +++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitterTest.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitter.java index 7272acdaf..a4c2f144c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitter.java @@ -1,7 +1,30 @@ package org.broadinstitute.dsm.model.elastic.filter; +import org.apache.commons.lang3.StringUtils; +import org.broadinstitute.dsm.model.Filter; +import org.broadinstitute.dsm.statics.ESObjectConstants; +import org.broadinstitute.dsm.util.ElasticSearchUtil; + public class JsonExtractSplitter extends EqualsSplitter { - //JSON_EXTRACT ( m.additional_values_json , '$.seeingIfBugExists' ) = 'true' + @Override + public String[] getValue() { + return new String[]{splittedFilter[1].trim()}; + } + + @Override + public String getInnerProperty() { + return ESObjectConstants.DYNAMIC_FIELDS; + } + @Override + protected String[] getFieldWithAlias() { + return splittedFilter[0] + .split(Filter.JSON_EXTRACT)[1] + .replace(Filter.OPEN_PARENTHESIS, StringUtils.EMPTY) + .replace(Filter.CLOSE_PARENTHESIS, StringUtils.EMPTY) + .trim() + .split(",")[0] + .split(ElasticSearchUtil.DOT_SEPARATOR); + } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitterTest.java new file mode 100644 index 000000000..dc18ce6d6 --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitterTest.java @@ -0,0 +1,30 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class JsonExtractSplitterTest { + + static String filter; + static BaseSplitter splitter; + + @BeforeClass + public static void setUp() { + filter = "JSON_EXTRACT ( m.additional_values_json , '$.seeingIfBugExists' ) = 'true'"; + splitter = SplitterFactory.createSplitter(Operator.JSON_EXTRACT); + splitter.setFilter(filter); + } + + @Test + public void getValue() { + Assert.assertEquals("'true'", splitter.getValue()[0]); + } + + @Test + public void getInnerProperty() { + Assert.assertEquals("dynamicFields", splitter.getInnerProperty()); + } +} \ No newline at end of file From fb4a6b3ed4e3979d63ff310751aa814cb7b0a2a1 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 21 Dec 2021 15:14:45 +0400 Subject: [PATCH 248/405] DDP-7121 | refactor json extractor to cover is not null case, refactor other classes as well according to changes required by json extractor --- .../dsm/model/elastic/Util.java | 4 +-- .../filter/CollectionQueryBuilder.java | 2 +- .../elastic/filter/JsonExtractSplitter.java | 33 ++++++++++++++----- .../model/elastic/filter/SplitterFactory.java | 10 ++++-- .../filter/CollectionQueryBuilderTest.java | 7 ++++ .../filter/DateGreaterSplitterTest.java | 4 +-- .../elastic/filter/DateLowerSplitterTest.java | 4 +-- .../elastic/filter/DateSplitterTest.java | 4 +-- .../filter/JsonExtractSplitterTest.java | 23 ++++++++----- .../filter/MultipleOptionsSplitterTest.java | 2 +- 10 files changed, 61 insertions(+), 32 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index d9d29cb00..fde95cf1b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -39,7 +39,7 @@ public class Util { public static final int FIRST_ELEMENT_INDEX = 0; public static final String UNDERSCORE_SEPARATOR = "_"; public static final String DOC = "_doc"; - private static final Pattern CAMEL_CASE_REGEX = Pattern.compile("(([a-z])+([A-z]))*"); + private static final Pattern CAMEL_CASE_REGEX = Pattern.compile("(([a-z])+([A-z])+(\\.)*)*"); private static final Pattern UPPER_CASE_REGEX = Pattern.compile("(?=\\p{Upper})"); public static final Gson GSON = new Gson(); @@ -73,7 +73,7 @@ public static String underscoresToCamelCase(String fieldName) { word.replace(FIRST_ELEMENT_INDEX, 1, String.valueOf(word.charAt(FIRST_ELEMENT_INDEX)).toUpperCase()); } } - return String.join("", words); + return String.join(StringUtils.EMPTY, words); } private static String handleAllUppercase(String word) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java index 0c962db97..482d5d3ce 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java @@ -35,7 +35,7 @@ public AbstractQueryBuilder build() { private void buildUpNestedQuery(BoolQueryBuilder boolQueryBuilder, List filterValues, FilterStrategy filterStrategy) { for (String filterValue : filterValues) { Operator operator = Operator.extract(filterValue); - BaseSplitter splitter = SplitterFactory.createSplitter(operator); + BaseSplitter splitter = SplitterFactory.createSplitter(operator, filterValue); splitter.setFilter(filterValue); String outerProperty = Util.TABLE_ALIAS_MAPPINGS.get(splitter.getAlias()).getPropertyName(); //medicalRecord String nestedPath = DSM_WITH_DOT + outerProperty; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitter.java index a4c2f144c..c13d666bc 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitter.java @@ -2,29 +2,44 @@ import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.Filter; +import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; -public class JsonExtractSplitter extends EqualsSplitter { +public class JsonExtractSplitter extends BaseSplitter { - @Override - public String[] getValue() { - return new String[]{splittedFilter[1].trim()}; + private BaseSplitter decoratedSplitter; + + public JsonExtractSplitter(BaseSplitter splitter) { + this.decoratedSplitter = splitter; + } + + public JsonExtractSplitter() { + decoratedSplitter = new EqualsSplitter(); } @Override - public String getInnerProperty() { - return ESObjectConstants.DYNAMIC_FIELDS; + public String[] split() { + decoratedSplitter.filter = filter; + return decoratedSplitter.split(); } @Override protected String[] getFieldWithAlias() { - return splittedFilter[0] + String[] splittedByJsonExtractAndComma = splittedFilter[0] .split(Filter.JSON_EXTRACT)[1] .replace(Filter.OPEN_PARENTHESIS, StringUtils.EMPTY) .replace(Filter.CLOSE_PARENTHESIS, StringUtils.EMPTY) .trim() - .split(",")[0] - .split(ElasticSearchUtil.DOT_SEPARATOR); + .split(","); + String[] splittedByDot = splittedByJsonExtractAndComma[0].split(ElasticSearchUtil.DOT_SEPARATOR); + String alias = splittedByDot[0]; + String removedSingleQuotes = splittedByJsonExtractAndComma[1] + .replace("'", StringUtils.EMPTY) + .trim(); + String innerProperty = removedSingleQuotes + .substring(removedSingleQuotes.indexOf(".")+1); + return new String[] {alias, String.join(".", ESObjectConstants.DYNAMIC_FIELDS, + innerProperty)}; } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java index fc38fdc46..ba208a4e7 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java @@ -2,7 +2,7 @@ public class SplitterFactory { - public static BaseSplitter createSplitter(Operator operator) { + public static BaseSplitter createSplitter(Operator operator, String filterValue) { BaseSplitter splitter; switch (operator) { case MULTIPLE_OPTIONS: @@ -36,7 +36,13 @@ public static BaseSplitter createSplitter(Operator operator) { splitter = new IsNotNullSplitter(); break; case JSON_EXTRACT: - splitter = new JsonExtractSplitter(); + Operator decoratedOperator = Operator.extract(filterValue); + if (Operator.IS_NOT_NULL.compareTo(decoratedOperator) != 0) { + splitter = new JsonExtractSplitter(); + break; + } + BaseSplitter decoratedSplitter = createSplitter(decoratedOperator, filterValue); + splitter = new JsonExtractSplitter(decoratedSplitter); break; default: throw new IllegalArgumentException("Unknown operator"); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java index e60a4e1e0..d219da066 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java @@ -142,6 +142,13 @@ public void dateGreaterBuild() { Assert.assertEquals(expected, actual); } + @Test + public void dynamicFieldsQueryBuild() { + + String filter = "AND JSON_EXTRACT ( m.additional_values_json , '$.seeingIfBugExists' ) = 'true' AND m.received <= STR_TO_DATE('2015-01-01', %yyyy-%MM-%dd)"; + + } + } \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateGreaterSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateGreaterSplitterTest.java index 21061c2ce..254d1f83e 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateGreaterSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateGreaterSplitterTest.java @@ -3,13 +3,11 @@ import org.junit.Assert; import org.junit.Test; -import static org.junit.Assert.*; - public class DateGreaterSplitterTest { @Test public void getValue() { - BaseSplitter splitter = SplitterFactory.createSplitter(Operator.DATE_GREATER); + BaseSplitter splitter = SplitterFactory.createSplitter(Operator.DATE_GREATER, ""); String filter = "AND m.mr_received >= STR_TO_DATE('2000-01-01','%Y-%m-%d')"; splitter.setFilter(filter); Assert.assertEquals("'2000-01-01'", splitter.getValue()[0]); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateLowerSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateLowerSplitterTest.java index 78e0a6279..a20fdee7b 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateLowerSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateLowerSplitterTest.java @@ -3,13 +3,11 @@ import org.junit.Assert; import org.junit.Test; -import static org.junit.Assert.*; - public class DateLowerSplitterTest { @Test public void getValue() { - BaseSplitter splitter = SplitterFactory.createSplitter(Operator.DATE_LESS); + BaseSplitter splitter = SplitterFactory.createSplitter(Operator.DATE_LESS, ""); String filter = "AND m.mr_received <= STR_TO_DATE('2000-01-01','%Y-%m-%d')"; splitter.setFilter(filter); Assert.assertEquals("'2000-01-01'", splitter.getValue()[0]); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitterTest.java index e36c2c14a..2f7f5f211 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitterTest.java @@ -3,14 +3,12 @@ import org.junit.Assert; import org.junit.Test; -import static org.junit.Assert.*; - public class DateSplitterTest { @Test public void split() { - BaseSplitter dateSplitter = SplitterFactory.createSplitter(Operator.DATE); + BaseSplitter dateSplitter = SplitterFactory.createSplitter(Operator.DATE, ""); dateSplitter.setFilter("STR_TO_DATE(m.fax_sent,'%Y-%m-%d') = STR_TO_DATE('2021-12-17','%Y-%m-%d')"); Assert.assertEquals("m", dateSplitter.getAlias()); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitterTest.java index dc18ce6d6..20ae479f2 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitterTest.java @@ -1,20 +1,20 @@ package org.broadinstitute.dsm.model.elastic.filter; +import org.apache.commons.lang3.StringUtils; import org.junit.Assert; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; -import static org.junit.Assert.*; - public class JsonExtractSplitterTest { - static String filter; - static BaseSplitter splitter; + String filter; + BaseSplitter splitter; - @BeforeClass - public static void setUp() { + @Before + public void setUp() { filter = "JSON_EXTRACT ( m.additional_values_json , '$.seeingIfBugExists' ) = 'true'"; - splitter = SplitterFactory.createSplitter(Operator.JSON_EXTRACT); + splitter = SplitterFactory.createSplitter(Operator.JSON_EXTRACT, filter); splitter.setFilter(filter); } @@ -25,6 +25,13 @@ public void getValue() { @Test public void getInnerProperty() { - Assert.assertEquals("dynamicFields", splitter.getInnerProperty()); + Assert.assertEquals("dynamicFields.seeingIfBugExists", splitter.getInnerProperty()); + } + + @Test + public void getIsNotNullValue() { + filter = "JSON_EXTRACT ( m.additional_values_json , '$.seeingIfBugExists' ) IS NOT NULL"; + splitter.setFilter(filter); + Assert.assertEquals(StringUtils.EMPTY, splitter.getValue()[0]); } } \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitterTest.java index a02fe2b4e..dac6247ce 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitterTest.java @@ -10,7 +10,7 @@ public class MultipleOptionsSplitterTest { public void getInnerProperty() { String filter = "( oD.fax_sent = 'review' OR oD.fax_sent = 'no' OR oD.fax_sent = 'hold' OR oD.fax_sent = 'request' OR oD.fax_sent = " + "'unable To Obtain' OR oD.fax_sent = 'sent' OR oD.fax_sent = 'received' OR oD.fax_sent = 'returned' )"; - BaseSplitter multipleSplitter = SplitterFactory.createSplitter(Operator.MULTIPLE_OPTIONS); + BaseSplitter multipleSplitter = SplitterFactory.createSplitter(Operator.MULTIPLE_OPTIONS, ""); multipleSplitter.setFilter(filter); assertEquals("faxSent", multipleSplitter.getInnerProperty()); } From c5e514097723c92e014ba994dc65dd244a7c952c Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 21 Dec 2021 16:45:24 +0400 Subject: [PATCH 249/405] DDP-7121 | convert true and false to boolean values, start handling special cases for JsonExtractSplitters --- .../dsm/model/elastic/export/parse/BaseParser.java | 4 ++-- .../dsm/model/elastic/filter/Operator.java | 14 +++++++++++++- .../model/elastic/filter/QueryBuilderFactory.java | 9 +++++++++ .../elastic/filter/CollectionQueryBuilderTest.java | 10 +++++++++- 4 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java index e3cbd0514..4d6e484d1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java @@ -68,9 +68,9 @@ protected boolean isBoolean(String value) { } public String convertBoolean(String value) { - if ("'1'".equals(value) || "NOT'0'".equals(value)) { + if ("'1'".equals(value) || "NOT'0'".equals(value) || "'true'".equals(value)) { return "true"; - } else if ("'0'".equals(value) || "NOT'1'".equals(value)) { + } else if ("'0'".equals(value) || "NOT'1'".equals(value) || "'false'".equals(value)) { return "false"; } else { return value; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java index 7fbec1716..daf8eb56d 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java @@ -1,7 +1,11 @@ package org.broadinstitute.dsm.model.elastic.filter; +import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.Filter; +import java.util.ArrayList; +import java.util.List; + public enum Operator { LIKE(Filter.LIKE_TRIMMED), EQUALS(Filter.EQUALS_TRIMMED), @@ -29,7 +33,15 @@ public static Operator getOperator(String value) { } public static Operator extract(String filter) { - if (filter.contains(Filter.IS_NOT_NULL_TRIMMED)) + String filterValue = StringUtils.EMPTY; + List chars = new ArrayList<>(); + for (int i = filter.length() - 1; i > 0 ; i--) { + if (Filter.IS_NOT_NULL_TRIMMED.equals(filterValue)) + break; + else + chars.add((char) filter.indexOf(i)); + } + if (new String(chars.toArray()).equals(Filter.IS_NOT_NULL_TRIMMED)) return IS_NOT_NULL; else if (filter.startsWith(Filter.OPEN_PARENTHESIS)) return MULTIPLE_OPTIONS; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java index 8336e288b..d14b7ff0b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java @@ -1,5 +1,6 @@ package org.broadinstitute.dsm.model.elastic.filter; +import org.apache.commons.lang3.StringUtils; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.ExistsQueryBuilder; import org.elasticsearch.index.query.MatchQueryBuilder; @@ -46,6 +47,14 @@ public static QueryBuilder buildQueryBuilder(Operator operator, QueryPayload pay } qb = boolQueryBuilder; break; + case JSON_EXTRACT: + Object[] dynamicFieldValues = payload.getValues(); + if (!StringUtils.EMPTY.equals(dynamicFieldValues[0])) { + qb = new MatchQueryBuilder(payload.getFieldName(), dynamicFieldValues[0]); + } else { + qb = new ExistsQueryBuilder(payload.getFieldName()); + } + break; default: throw new IllegalArgumentException("Unknown operator"); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java index d219da066..ec61083a9 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java @@ -145,8 +145,16 @@ public void dateGreaterBuild() { @Test public void dynamicFieldsQueryBuild() { - String filter = "AND JSON_EXTRACT ( m.additional_values_json , '$.seeingIfBugExists' ) = 'true' AND m.received <= STR_TO_DATE('2015-01-01', %yyyy-%MM-%dd)"; + String filter = "AND JSON_EXTRACT ( m.additional_values_json , '$.seeingIfBugExists' ) = 'true' AND JSON_EXTRACT ( m.additional_values_json , '$.tryAgain' ) IS NOT NULL"; + collectionQueryBuilder.setFilter(filter); + AbstractQueryBuilder actual = collectionQueryBuilder.build(); + + AbstractQueryBuilder expected = new BoolQueryBuilder().must(new NestedQueryBuilder("dsm.medicalRecord", + new MatchQueryBuilder("dsm.medicalRecord.dynamicFields.seeingIfBugExists", true), ScoreMode.Avg)) + .must(new NestedQueryBuilder("dsm.medicalRecord", new ExistsQueryBuilder("dsm.medicalRecord.dynamicFields.tryAgain"),ScoreMode.Avg)); + + Assert.assertEquals(expected, actual); } From 419ab0350af96509e55132fe8dddfb76d90c4369 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 21 Dec 2021 18:13:51 +0400 Subject: [PATCH 250/405] DDP-7121 | refactoring for json extract --- .../dsm/model/elastic/filter/Operator.java | 10 +--------- .../dsm/model/elastic/filter/SplitterFactory.java | 5 ++++- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java index daf8eb56d..c0f4a7f42 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java @@ -33,15 +33,7 @@ public static Operator getOperator(String value) { } public static Operator extract(String filter) { - String filterValue = StringUtils.EMPTY; - List chars = new ArrayList<>(); - for (int i = filter.length() - 1; i > 0 ; i--) { - if (Filter.IS_NOT_NULL_TRIMMED.equals(filterValue)) - break; - else - chars.add((char) filter.indexOf(i)); - } - if (new String(chars.toArray()).equals(Filter.IS_NOT_NULL_TRIMMED)) + if (filter.endsWith(Filter.IS_NOT_NULL_TRIMMED) && !filter.startsWith(Filter.JSON_EXTRACT)) return IS_NOT_NULL; else if (filter.startsWith(Filter.OPEN_PARENTHESIS)) return MULTIPLE_OPTIONS; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java index ba208a4e7..6c9e9c141 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java @@ -1,5 +1,8 @@ package org.broadinstitute.dsm.model.elastic.filter; +import org.apache.commons.lang3.StringUtils; +import org.broadinstitute.dsm.model.Filter; + public class SplitterFactory { public static BaseSplitter createSplitter(Operator operator, String filterValue) { @@ -36,7 +39,7 @@ public static BaseSplitter createSplitter(Operator operator, String filterValue) splitter = new IsNotNullSplitter(); break; case JSON_EXTRACT: - Operator decoratedOperator = Operator.extract(filterValue); + Operator decoratedOperator = Operator.extract(filterValue.replace(Filter.JSON_EXTRACT, StringUtils.EMPTY)); if (Operator.IS_NOT_NULL.compareTo(decoratedOperator) != 0) { splitter = new JsonExtractSplitter(); break; From b6b2dd2a42f43f40175dad56ce04afc4c2c0e3ad Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 22 Dec 2021 11:15:59 +0400 Subject: [PATCH 251/405] DDP-7121 | refactor dsmabstractquerybuilder to make it more generalized --- .../filter/CollectionQueryBuilder.java | 52 +------------- .../model/elastic/filter/DateSplitter.java | 1 - .../filter/DsmAbstractQueryBuilder.java | 71 ++++++++++++++++++- .../model/elastic/filter/EqualsSplitter.java | 1 - .../elastic/filter/DateSplitterTest.java | 4 +- 5 files changed, 74 insertions(+), 55 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java index 482d5d3ce..5c3746614 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java @@ -1,13 +1,8 @@ package org.broadinstitute.dsm.model.elastic.filter; -import org.apache.commons.lang3.StringUtils; import org.apache.lucene.search.join.ScoreMode; -import org.broadinstitute.dsm.model.Filter; -import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; -import org.elasticsearch.index.query.*; - -import java.util.*; +import org.elasticsearch.index.query.NestedQueryBuilder; public class CollectionQueryBuilder extends DsmAbstractQueryBuilder { @@ -18,50 +13,9 @@ public CollectionQueryBuilder(String filter, Parser parser) { public CollectionQueryBuilder() {} @Override - public AbstractQueryBuilder build() { - Map> parsedFilters = parseFiltersByLogicalOperators(); - BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); - for (Map.Entry> parsedFilter: parsedFilters.entrySet()) { - List filterValues = parsedFilter.getValue(); - if (parsedFilter.getKey().equals(Filter.AND_TRIMMED)) { - buildUpNestedQuery(boolQueryBuilder, filterValues, BoolQueryBuilder::must); - } else { - buildUpNestedQuery(boolQueryBuilder, filterValues, BoolQueryBuilder::should); - } - } - return boolQueryBuilder; - } - - private void buildUpNestedQuery(BoolQueryBuilder boolQueryBuilder, List filterValues, FilterStrategy filterStrategy) { - for (String filterValue : filterValues) { - Operator operator = Operator.extract(filterValue); - BaseSplitter splitter = SplitterFactory.createSplitter(operator, filterValue); - splitter.setFilter(filterValue); - String outerProperty = Util.TABLE_ALIAS_MAPPINGS.get(splitter.getAlias()).getPropertyName(); //medicalRecord - String nestedPath = DSM_WITH_DOT + outerProperty; - QueryPayload queryPayload = new QueryPayload(nestedPath, splitter.getInnerProperty(), parser.parse(splitter.getValue())); - QueryBuilder queryBuilder = QueryBuilderFactory.buildQueryBuilder(operator, queryPayload); - filterStrategy.build(boolQueryBuilder, new NestedQueryBuilder(nestedPath, queryBuilder, ScoreMode.Avg)); - } + protected void buildEachQuery(FilterStrategy filterStrategy) { + filterStrategy.build(boolQueryBuilder, new NestedQueryBuilder(buildPath(), queryBuilder, ScoreMode.Avg)); } - protected Map> parseFiltersByLogicalOperators() { - Map> filterByLogicalOperators = new HashMap<>(Map.of(Filter.AND_TRIMMED, new ArrayList<>(), Filter.OR_TRIMMED, - new ArrayList<>())); - String[] andSeparated = filter.split(Filter.AND_TRIMMED); - for (String eachFilter : andSeparated) { - String cleanedEachFilter = eachFilter.trim(); - if (cleanedEachFilter.contains(Filter.OR_TRIMMED) && !cleanedEachFilter.startsWith(Filter.OPEN_PARENTHESIS) && !cleanedEachFilter.endsWith(Filter.CLOSE_PARENTHESIS)) { - String[] orSeparated = cleanedEachFilter.split(Filter.OR_TRIMMED); - filterByLogicalOperators.get(Filter.AND_TRIMMED).add(orSeparated[0].trim()); - Arrays.stream(orSeparated) - .skip(1) - .forEach(f -> filterByLogicalOperators.get(Filter.OR_TRIMMED).add(f.trim())); - } else if (StringUtils.isNotBlank(cleanedEachFilter)){ - filterByLogicalOperators.get(Filter.AND_TRIMMED).add(cleanedEachFilter); - } - } - return filterByLogicalOperators; - } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitter.java index 2bcba02e8..db77f7749 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitter.java @@ -5,7 +5,6 @@ public class DateSplitter extends BaseSplitter { - @Override public String[] split() { // STR_TO_DATE(m.fax_sent,'%Y-%m-%d') = STR_TO_DATE('2021-12-17','%Y-%m-%d') diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java index 8d2f6e2dc..2ec2c3118 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java @@ -1,22 +1,41 @@ package org.broadinstitute.dsm.model.elastic.filter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.lang3.StringUtils; +import org.apache.lucene.search.join.ScoreMode; +import org.broadinstitute.dsm.model.Filter; +import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.statics.ESObjectConstants; import org.elasticsearch.index.query.AbstractQueryBuilder; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.NestedQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; public abstract class DsmAbstractQueryBuilder { protected static final String DSM_WITH_DOT = ESObjectConstants.DSM + DBConstants.ALIAS_DELIMITER; protected String filter; protected Parser parser; + protected BoolQueryBuilder boolQueryBuilder; + protected QueryBuilder queryBuilder; + protected BaseSplitter splitter; public DsmAbstractQueryBuilder(String filter, Parser parser) { + this(); this.parser = parser; this.filter = filter; } - public DsmAbstractQueryBuilder() {} + public DsmAbstractQueryBuilder() { + boolQueryBuilder = new BoolQueryBuilder(); + } public void setFilter(String filter) { this.filter = filter; @@ -26,5 +45,53 @@ public void setParser(Parser parser) { this.parser = parser; } - public abstract AbstractQueryBuilder build(); + public AbstractQueryBuilder build() { + Map> parsedFilters = parseFiltersByLogicalOperators(); + for (Map.Entry> parsedFilter: parsedFilters.entrySet()) { + List filterValues = parsedFilter.getValue(); + if (parsedFilter.getKey().equals(Filter.AND_TRIMMED)) { + buildUpQuery(filterValues, BoolQueryBuilder::must); + } else { + buildUpQuery(filterValues, BoolQueryBuilder::should); + } + } + return boolQueryBuilder; + } + + protected void buildUpQuery(List filterValues, FilterStrategy filterStrategy) { + for (String filterValue : filterValues) { + Operator operator = Operator.extract(filterValue); + splitter = SplitterFactory.createSplitter(operator, filterValue); + splitter.setFilter(filterValue); + QueryPayload queryPayload = new QueryPayload(buildPath(), splitter.getInnerProperty(), parser.parse(splitter.getValue())); + queryBuilder = QueryBuilderFactory.buildQueryBuilder(operator, queryPayload); + buildEachQuery(filterStrategy); + } + } + + protected String buildPath() { + String outerProperty = Util.TABLE_ALIAS_MAPPINGS.get(splitter.getAlias()).getPropertyName(); //medicalRecord + return DSM_WITH_DOT + outerProperty; + } + + protected abstract void buildEachQuery(FilterStrategy filterStrategy); + + protected Map> parseFiltersByLogicalOperators() { + Map> filterByLogicalOperators = new HashMap<>(Map.of(Filter.AND_TRIMMED, new ArrayList<>(), Filter.OR_TRIMMED, + new ArrayList<>())); + String[] andSeparated = filter.split(Filter.AND_TRIMMED); + for (String eachFilter : andSeparated) { + String cleanedEachFilter = eachFilter.trim(); + if (cleanedEachFilter.contains(Filter.OR_TRIMMED) && !cleanedEachFilter.startsWith(Filter.OPEN_PARENTHESIS) && !cleanedEachFilter.endsWith(Filter.CLOSE_PARENTHESIS)) { + String[] orSeparated = cleanedEachFilter.split(Filter.OR_TRIMMED); + filterByLogicalOperators.get(Filter.AND_TRIMMED).add(orSeparated[0].trim()); + Arrays.stream(orSeparated) + .skip(1) + .forEach(f -> filterByLogicalOperators.get(Filter.OR_TRIMMED).add(f.trim())); + } else if (StringUtils.isNotBlank(cleanedEachFilter)){ + filterByLogicalOperators.get(Filter.AND_TRIMMED).add(cleanedEachFilter); + } + } + return filterByLogicalOperators; + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/EqualsSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/EqualsSplitter.java index 2035999fd..0f5867c6c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/EqualsSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/EqualsSplitter.java @@ -4,7 +4,6 @@ public class EqualsSplitter extends BaseSplitter { - @Override public String[] split() { return filter.split(Filter.EQUALS_TRIMMED); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitterTest.java index 2f7f5f211..0db186615 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitterTest.java @@ -8,8 +8,8 @@ public class DateSplitterTest { @Test public void split() { - BaseSplitter dateSplitter = SplitterFactory.createSplitter(Operator.DATE, ""); - dateSplitter.setFilter("STR_TO_DATE(m.fax_sent,'%Y-%m-%d') = STR_TO_DATE('2021-12-17','%Y-%m-%d')"); + String filter = "STR_TO_DATE(m.fax_sent,'%Y-%m-%d') = STR_TO_DATE('2021-12-17','%Y-%m-%d')"; + BaseSplitter dateSplitter = SplitterFactory.createSplitter(Operator.DATE, filter); Assert.assertEquals("m", dateSplitter.getAlias()); Assert.assertEquals("faxSent", dateSplitter.getInnerProperty()); From 089299437c2bd1c514a072219f63ba42fc5145b7 Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 22 Dec 2021 11:53:18 +0400 Subject: [PATCH 252/405] DDP-7121 | make tests pass --- .../model/elastic/filter/CollectionQueryBuilderTest.java | 7 ++++--- .../dsm/model/elastic/filter/DateSplitterTest.java | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java index ec61083a9..ddf9af069 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java @@ -12,15 +12,16 @@ import org.elasticsearch.index.query.NestedQueryBuilder; import org.elasticsearch.index.query.RangeQueryBuilder; import org.junit.Assert; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; public class CollectionQueryBuilderTest { - static CollectionQueryBuilder collectionQueryBuilder; + CollectionQueryBuilder collectionQueryBuilder; - @BeforeClass - public static void setUp() { + @Before + public void setUp() { collectionQueryBuilder = new CollectionQueryBuilder(); collectionQueryBuilder.setParser(new FilterParser()); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitterTest.java index 0db186615..a93f76d16 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitterTest.java @@ -10,6 +10,7 @@ public void split() { String filter = "STR_TO_DATE(m.fax_sent,'%Y-%m-%d') = STR_TO_DATE('2021-12-17','%Y-%m-%d')"; BaseSplitter dateSplitter = SplitterFactory.createSplitter(Operator.DATE, filter); + dateSplitter.setFilter(filter); Assert.assertEquals("m", dateSplitter.getAlias()); Assert.assertEquals("faxSent", dateSplitter.getInnerProperty()); From 1e7b768e02325e55df9390ab6bb3ba0c7d0a736c Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 22 Dec 2021 12:10:59 +0400 Subject: [PATCH 253/405] DDP-7121 | implement SingleQueryBuilder, make tests pass --- .../filter/DsmAbstractQueryBuilder.java | 13 ++----- .../elastic/filter/SingleQueryBuilder.java | 11 ++++++ .../filter/SingleQueryBuilderTest.java | 34 +++++++++++++++++++ 3 files changed, 48 insertions(+), 10 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/SingleQueryBuilder.java create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/filter/SingleQueryBuilderTest.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java index 2ec2c3118..6719f24f0 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java @@ -1,13 +1,6 @@ package org.broadinstitute.dsm.model.elastic.filter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - import org.apache.commons.lang3.StringUtils; -import org.apache.lucene.search.join.ScoreMode; import org.broadinstitute.dsm.model.Filter; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; @@ -15,9 +8,10 @@ import org.broadinstitute.dsm.statics.ESObjectConstants; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.NestedQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; +import java.util.*; + public abstract class DsmAbstractQueryBuilder { protected static final String DSM_WITH_DOT = ESObjectConstants.DSM + DBConstants.ALIAS_DELIMITER; @@ -70,8 +64,7 @@ protected void buildUpQuery(List filterValues, FilterStrategy filterStra } protected String buildPath() { - String outerProperty = Util.TABLE_ALIAS_MAPPINGS.get(splitter.getAlias()).getPropertyName(); //medicalRecord - return DSM_WITH_DOT + outerProperty; + return DSM_WITH_DOT + Util.TABLE_ALIAS_MAPPINGS.get(splitter.getAlias()).getPropertyName(); } protected abstract void buildEachQuery(FilterStrategy filterStrategy); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SingleQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SingleQueryBuilder.java new file mode 100644 index 000000000..6f62eca5e --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SingleQueryBuilder.java @@ -0,0 +1,11 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import org.elasticsearch.index.query.MatchQueryBuilder; + +public class SingleQueryBuilder extends DsmAbstractQueryBuilder { + + @Override + protected void buildEachQuery(FilterStrategy filterStrategy) { + filterStrategy.build(boolQueryBuilder, queryBuilder); + } +} diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/SingleQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/SingleQueryBuilderTest.java new file mode 100644 index 000000000..3227142ab --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/SingleQueryBuilderTest.java @@ -0,0 +1,34 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import org.elasticsearch.index.query.AbstractQueryBuilder; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.MatchQueryBuilder; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class SingleQueryBuilderTest { + + SingleQueryBuilder singleQueryBuilder; + + @Before + public void setUp() { + singleQueryBuilder = new SingleQueryBuilder(); + singleQueryBuilder.setParser(new FilterParser()); + } + + @Test + public void buildEachQuery() { + + String filter = "AND p.participantId = '1234'"; + singleQueryBuilder.setFilter(filter); + + AbstractQueryBuilder actual = singleQueryBuilder.build(); + BoolQueryBuilder expected = new BoolQueryBuilder().must(new MatchQueryBuilder("dsm.participant.participantId", "1234")); + + Assert.assertEquals(expected, actual); + } + +} \ No newline at end of file From 81642204615c87fb2ca198b9d831ba89f7005f32 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 22 Dec 2021 16:16:07 +0400 Subject: [PATCH 254/405] DDP-7121 | refactor extract method of operator to filter emplty strings, passing queryConditions directly instead of merge conditions --- .../broadinstitute/dsm/model/elastic/filter/Operator.java | 7 ++++++- .../dsm/model/elastic/filter/SingleQueryBuilder.java | 7 +++++++ .../filter/participant/BaseFilterParticipantList.java | 5 +++-- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java index c0f4a7f42..2292276d5 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java @@ -4,7 +4,9 @@ import org.broadinstitute.dsm.model.Filter; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; public enum Operator { LIKE(Filter.LIKE_TRIMMED), @@ -45,7 +47,10 @@ else if (filter.contains(Filter.DATE_LESS)) return DATE_LESS; else if (filter.contains(Filter.JSON_EXTRACT)) return JSON_EXTRACT; - String operator = filter.split(" ")[1]; + String operator = Arrays.stream(filter.split(" ")) + .filter(StringUtils::isNotBlank) + .collect(Collectors.toList()) + .get(1); return getOperator(operator); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SingleQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SingleQueryBuilder.java index 6f62eca5e..0a6f3b201 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SingleQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SingleQueryBuilder.java @@ -1,9 +1,16 @@ package org.broadinstitute.dsm.model.elastic.filter; +import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import org.elasticsearch.index.query.MatchQueryBuilder; public class SingleQueryBuilder extends DsmAbstractQueryBuilder { + public SingleQueryBuilder(String filter, Parser parser) { + super(filter, parser); + } + + public SingleQueryBuilder() {} + @Override protected void buildEachQuery(FilterStrategy filterStrategy) { filterStrategy.build(boolQueryBuilder, queryBuilder); diff --git a/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java b/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java index 803211b1f..ec1fe31c0 100644 --- a/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java +++ b/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java @@ -118,8 +118,9 @@ protected ParticipantWrapperResult filterParticipantList(Filter[] filters, Map instead of passing mergeConditions passing queryConditions directly to avoid merge different keys like o,r -> p and + // etc. + return new ParticipantWrapper(participantWrapperPayload.withFilter(queryConditions).build(), elasticSearch).getFilteredList(); } else { return new ParticipantWrapper(participantWrapperPayload.build(), elasticSearch).getFilteredList(); } From 1a16dcd9c9a37e119634c0665ed97969a93fb9b2 Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 22 Dec 2021 17:27:01 +0400 Subject: [PATCH 255/405] DDP-7121 | start merging two different queries from two different sources --- .../filter/DsmAbstractQueryBuilder.java | 7 ++ .../BaseFilterParticipantList.java | 22 ++++--- .../model/participant/ParticipantWrapper.java | 65 +++++++++++++------ 3 files changed, 65 insertions(+), 29 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java index 6719f24f0..573a24fba 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java @@ -3,6 +3,7 @@ import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.Filter; import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.statics.ESObjectConstants; @@ -27,6 +28,12 @@ public DsmAbstractQueryBuilder(String filter, Parser parser) { this.filter = filter; } + public static DsmAbstractQueryBuilder of(String alias) { + DsmAbstractQueryBuilder queryBuilder; + boolean isCollection = Util.TABLE_ALIAS_MAPPINGS.get(alias).isCollection(); + return isCollection ? new CollectionQueryBuilder() : new SingleQueryBuilder(); + } + public DsmAbstractQueryBuilder() { boolQueryBuilder = new BoolQueryBuilder(); } diff --git a/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java b/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java index ec1fe31c0..62ee7b1d9 100644 --- a/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java +++ b/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java @@ -104,13 +104,19 @@ protected ParticipantWrapperResult filterParticipantList(Filter[] filters, Map mergeConditions = new HashMap<>(); for (String filter : queryConditions.keySet()) { - if (DBConstants.DDP_PARTICIPANT_RECORD_ALIAS.equals(filter) - || DBConstants.DDP_PARTICIPANT_EXIT_ALIAS.equals(filter) || DBConstants.DDP_ONC_HISTORY_ALIAS.equals(filter)) { - mergeConditions.merge(DBConstants.DDP_PARTICIPANT_ALIAS, queryConditions.get(filter), String::concat); - } else if (DBConstants.DDP_INSTITUTION_ALIAS.equals(filter)) { - mergeConditions.merge(DBConstants.DDP_MEDICAL_RECORD_ALIAS, queryConditions.get(filter), String::concat); - } else if (DBConstants.DDP_TISSUE_ALIAS.equals(filter)) { - mergeConditions.merge(DBConstants.DDP_ONC_HISTORY_DETAIL_ALIAS, queryConditions.get(filter), String::concat); +// if (DBConstants.DDP_PARTICIPANT_RECORD_ALIAS.equals(filter) +// || DBConstants.DDP_PARTICIPANT_EXIT_ALIAS.equals(filter) || DBConstants.DDP_ONC_HISTORY_ALIAS.equals(filter)) { +// mergeConditions.merge(DBConstants.DDP_PARTICIPANT_ALIAS, queryConditions.get(filter), String::concat); +// } else if (DBConstants.DDP_INSTITUTION_ALIAS.equals(filter)) { +// mergeConditions.merge(DBConstants.DDP_MEDICAL_RECORD_ALIAS, queryConditions.get(filter), String::concat); +// } else if (DBConstants.DDP_TISSUE_ALIAS.equals(filter)) { +// mergeConditions.merge(DBConstants.DDP_ONC_HISTORY_DETAIL_ALIAS, queryConditions.get(filter), String::concat); +// } else { +// mergeConditions.merge(filter, queryConditions.get(filter), String::concat); +// } + if (DBConstants.DDP_PARTICIPANT_EXIT_ALIAS.equals(filter)) { + String exitFilter = queryConditions.get(filter).replace("ex.", "p."); + mergeConditions.merge(DBConstants.DDP_PARTICIPANT_ALIAS, exitFilter, String::concat); } else { mergeConditions.merge(filter, queryConditions.get(filter), String::concat); } @@ -120,7 +126,7 @@ protected ParticipantWrapperResult filterParticipantList(Filter[] filters, Map instead of passing mergeConditions passing queryConditions directly to avoid merge different keys like o,r -> p and // etc. - return new ParticipantWrapper(participantWrapperPayload.withFilter(queryConditions).build(), elasticSearch).getFilteredList(); + return new ParticipantWrapper(participantWrapperPayload.withFilter(mergeConditions).build(), elasticSearch).getFilteredList(); } else { return new ParticipantWrapper(participantWrapperPayload.build(), elasticSearch).getFilteredList(); } diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java index 1724e13b1..b47300d8c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java @@ -11,12 +11,17 @@ import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; import org.broadinstitute.dsm.model.elastic.ESDsm; import org.broadinstitute.dsm.model.elastic.ESProfile; +import org.broadinstitute.dsm.model.elastic.filter.DsmAbstractQueryBuilder; +import org.broadinstitute.dsm.model.elastic.filter.FilterParser; import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchable; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import org.broadinstitute.dsm.model.at.DefaultValues; import org.broadinstitute.dsm.statics.DBConstants; +import org.broadinstitute.dsm.util.ElasticSearchUtil; +import org.elasticsearch.index.query.AbstractQueryBuilder; +import org.elasticsearch.index.query.BoolQueryBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -82,26 +87,36 @@ public ParticipantWrapperResult getFilteredList() { //TODO could be better, good place for refactoring for future private void fetchAndPrepareDataByFilters(DDPInstance ddpInstance, Map filters) { List participantIdsToFetch = Collections.emptyList(); + FilterParser parser = new FilterParser(); + BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); for (String source : filters.keySet()) { if (StringUtils.isNotBlank(filters.get(source))) { - if (DBConstants.DDP_PARTICIPANT_ALIAS.equals(source)) { - Map participants = - Participant.getParticipants(ddpInstance.getName(), filters.get(source)); - this.participants = new ArrayList<>(participants.values()); - participantIdsToFetch = new ArrayList<>(participants.keySet()); - } - else if (DBConstants.DDP_MEDICAL_RECORD_ALIAS.equals(source)) { - medicalRecords = MedicalRecord.getMedicalRecords(ddpInstance.getName(), filters.get(source)); - participantIdsToFetch = new ArrayList<>(medicalRecords.keySet()); - } - else if (DBConstants.DDP_ONC_HISTORY_DETAIL_ALIAS.equals(source)) { - oncHistoryDetails = OncHistoryDetail.getOncHistoryDetails(ddpInstance.getName(), filters.get(source)); - participantIdsToFetch = new ArrayList<>(oncHistoryDetails.keySet()); - } - else if (DBConstants.DDP_KIT_REQUEST_ALIAS.equals(source)) { - kitRequests = KitRequestShipping.getKitRequests(ddpInstance, filters.get(source)); - participantIdsToFetch = new ArrayList<>(kitRequests.keySet()); + if (isUnderDsmKey(source)) { + DsmAbstractQueryBuilder queryBuilder = DsmAbstractQueryBuilder.of(source); + queryBuilder.setFilter(filters.get(source)); + queryBuilder.setParser(parser); + boolQueryBuilder.must(queryBuilder.build()); } + +// if (DBConstants.DDP_PARTICIPANT_ALIAS.equals(source)) { +// Map participants = +// Participant.getParticipants(ddpInstance.getName(), filters.get(source)); +// this.participants = new ArrayList<>(participants.values()); +// participantIdsToFetch = new ArrayList<>(participants.keySet()); +// } +// else if (DBConstants.DDP_MEDICAL_RECORD_ALIAS.equals(source)) { +// medicalRecords = MedicalRecord.getMedicalRecords(ddpInstance.getName(), filters.get(source)); +// participantIdsToFetch = new ArrayList<>(medicalRecords.keySet()); +// } +// else if (DBConstants.DDP_ONC_HISTORY_DETAIL_ALIAS.equals(source)) { +// oncHistoryDetails = OncHistoryDetail.getOncHistoryDetails(ddpInstance.getName(), filters.get(source)); +// participantIdsToFetch = new ArrayList<>(oncHistoryDetails.keySet()); +// } +// else if (DBConstants.DDP_KIT_REQUEST_ALIAS.equals(source)) { +// kitRequests = KitRequestShipping.getKitRequests(ddpInstance, filters.get(source)); +// participantIdsToFetch = new ArrayList<>(kitRequests.keySet()); +// } + else if (DBConstants.DDP_PARTICIPANT_DATA_ALIAS.equals(source)) { participantData = new ParticipantDataDao().getParticipantDataByInstanceIdAndFilterQuery(Integer.parseInt(ddpInstance.getDdpInstanceId()), filters.get(source)); participantIdsToFetch = new ArrayList<>(participantData.keySet()); @@ -113,16 +128,17 @@ else if (DBConstants.DDP_PARTICIPANT_DATA_ALIAS.equals(source)) { participantData = defaultValues.addDefaultValues(); } } - else if (DBConstants.DDP_ABSTRACTION_ALIAS.equals(source)) { - abstractionActivities = AbstractionActivity.getAllAbstractionActivityByRealm(ddpInstance.getName(), filters.get(source)); - participantIdsToFetch = new ArrayList<>(abstractionActivities.keySet()); - } +// else if (DBConstants.DDP_ABSTRACTION_ALIAS.equals(source)) { +// abstractionActivities = AbstractionActivity.getAllAbstractionActivityByRealm(ddpInstance.getName(), filters.get(source)); +// participantIdsToFetch = new ArrayList<>(abstractionActivities.keySet()); +// } else { //source is not of any study-manager table so it must be ES esData = elasticSearchable.getParticipantsByRangeAndFilter(ddpInstance.getParticipantIndexES(), participantWrapperPayload.getFrom(), participantWrapperPayload.getTo(), filters.get(source)); participantIdsToFetch = esData.getEsParticipants().stream().map(ElasticSearchParticipantDto::getParticipantId) .collect( Collectors.toList()); + boolQueryBuilder.must(ElasticSearchUtil.createESQuery(filters.get(source))); } } } @@ -164,6 +180,13 @@ else if (DBConstants.DDP_ABSTRACTION_ALIAS.equals(source)) { if (proxiesByParticipantIds.isEmpty()) { proxiesByParticipantIds = getProxiesWithParticipantIdsFromElasticList(ddpInstance.getUsersIndexES(), esData.getEsParticipants()); } + + System.out.println(); + System.out.println("awdwa"); + } + + private boolean isUnderDsmKey(String source) { + return DBConstants.DDP_PARTICIPANT_ALIAS.equals(source) || DBConstants.DDP_MEDICAL_RECORD_ALIAS.equals(source) || DBConstants.DDP_ONC_HISTORY_DETAIL_ALIAS.equals(source) || DBConstants.DDP_KIT_REQUEST_ALIAS.equals(source) || DBConstants.DDP_TISSUE_ALIAS.equals(source) || DBConstants.DDP_ONC_HISTORY_ALIAS.equals(source) || DBConstants.DDP_PARTICIPANT_RECORD_ALIAS.equals(source); } private void fetchAndPrepareData(DDPInstance ddpInstance) { From 71f2a45ab52d731eef173eedad8f5998f44a41af Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 22 Dec 2021 19:07:24 +0400 Subject: [PATCH 256/405] DDP-7121 | add method to clean up meaningless texts from filter --- .../model/elastic/filter/DsmAbstractQueryBuilder.java | 8 +++++++- .../dsm/model/elastic/filter/Operator.java | 1 + .../dsm/model/participant/ParticipantWrapper.java | 10 ++++++++-- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java index 573a24fba..9cb9e4174 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java @@ -12,6 +12,7 @@ import org.elasticsearch.index.query.QueryBuilder; import java.util.*; +import java.util.regex.Pattern; public abstract class DsmAbstractQueryBuilder { @@ -81,7 +82,7 @@ protected Map> parseFiltersByLogicalOperators() { new ArrayList<>())); String[] andSeparated = filter.split(Filter.AND_TRIMMED); for (String eachFilter : andSeparated) { - String cleanedEachFilter = eachFilter.trim(); + String cleanedEachFilter = cleanUpData(eachFilter).trim(); if (cleanedEachFilter.contains(Filter.OR_TRIMMED) && !cleanedEachFilter.startsWith(Filter.OPEN_PARENTHESIS) && !cleanedEachFilter.endsWith(Filter.CLOSE_PARENTHESIS)) { String[] orSeparated = cleanedEachFilter.split(Filter.OR_TRIMMED); filterByLogicalOperators.get(Filter.AND_TRIMMED).add(orSeparated[0].trim()); @@ -94,4 +95,9 @@ protected Map> parseFiltersByLogicalOperators() { } return filterByLogicalOperators; } + + private String cleanUpData(String filterValue) { + final Pattern additionalValuesJsonToClean = Pattern.compile("([a-zA-Z]+\\.(additional_values_json IS NOT NULL))"); + return filterValue.replaceAll(additionalValuesJsonToClean.pattern(), StringUtils.EMPTY); + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java index 2292276d5..788c46cc4 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java @@ -35,6 +35,7 @@ public static Operator getOperator(String value) { } public static Operator extract(String filter) { + filter = filter.trim(); if (filter.endsWith(Filter.IS_NOT_NULL_TRIMMED) && !filter.startsWith(Filter.JSON_EXTRACT)) return IS_NOT_NULL; else if (filter.startsWith(Filter.OPEN_PARENTHESIS)) diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java index b47300d8c..b1da40d4e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java @@ -132,7 +132,7 @@ else if (DBConstants.DDP_PARTICIPANT_DATA_ALIAS.equals(source)) { // abstractionActivities = AbstractionActivity.getAllAbstractionActivityByRealm(ddpInstance.getName(), filters.get(source)); // participantIdsToFetch = new ArrayList<>(abstractionActivities.keySet()); // } - else { //source is not of any study-manager table so it must be ES + else if ("ES".equals(source)){ //source is not of any study-manager table so it must be ES esData = elasticSearchable.getParticipantsByRangeAndFilter(ddpInstance.getParticipantIndexES(), participantWrapperPayload.getFrom(), participantWrapperPayload.getTo(), filters.get(source)); participantIdsToFetch = esData.getEsParticipants().stream().map(ElasticSearchParticipantDto::getParticipantId) @@ -186,7 +186,13 @@ else if (DBConstants.DDP_PARTICIPANT_DATA_ALIAS.equals(source)) { } private boolean isUnderDsmKey(String source) { - return DBConstants.DDP_PARTICIPANT_ALIAS.equals(source) || DBConstants.DDP_MEDICAL_RECORD_ALIAS.equals(source) || DBConstants.DDP_ONC_HISTORY_DETAIL_ALIAS.equals(source) || DBConstants.DDP_KIT_REQUEST_ALIAS.equals(source) || DBConstants.DDP_TISSUE_ALIAS.equals(source) || DBConstants.DDP_ONC_HISTORY_ALIAS.equals(source) || DBConstants.DDP_PARTICIPANT_RECORD_ALIAS.equals(source); + return DBConstants.DDP_PARTICIPANT_ALIAS.equals(source) + || DBConstants.DDP_MEDICAL_RECORD_ALIAS.equals(source) + || DBConstants.DDP_ONC_HISTORY_DETAIL_ALIAS.equals(source) + || DBConstants.DDP_KIT_REQUEST_ALIAS.equals(source) + || DBConstants.DDP_TISSUE_ALIAS.equals(source) + || DBConstants.DDP_ONC_HISTORY_ALIAS.equals(source) + || DBConstants.DDP_PARTICIPANT_RECORD_ALIAS.equals(source); } private void fetchAndPrepareData(DDPInstance ddpInstance) { From f7190797fa4a0c9ce19c41b14aef82d604b073e4 Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 23 Dec 2021 17:58:20 +0400 Subject: [PATCH 257/405] DDP-7121 | start handling the special case related to "OR" and "AND" operators in filter values --- .../filter/DsmAbstractQueryBuilder.java | 90 ++++++++++++++++--- .../dsm/model/elastic/filter/Operator.java | 2 - .../filter/CollectionQueryBuilderTest.java | 15 +++- 3 files changed, 89 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java index 9cb9e4174..9521da0cf 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java @@ -78,21 +78,85 @@ protected String buildPath() { protected abstract void buildEachQuery(FilterStrategy filterStrategy); protected Map> parseFiltersByLogicalOperators() { - Map> filterByLogicalOperators = new HashMap<>(Map.of(Filter.AND_TRIMMED, new ArrayList<>(), Filter.OR_TRIMMED, - new ArrayList<>())); - String[] andSeparated = filter.split(Filter.AND_TRIMMED); - for (String eachFilter : andSeparated) { - String cleanedEachFilter = cleanUpData(eachFilter).trim(); - if (cleanedEachFilter.contains(Filter.OR_TRIMMED) && !cleanedEachFilter.startsWith(Filter.OPEN_PARENTHESIS) && !cleanedEachFilter.endsWith(Filter.CLOSE_PARENTHESIS)) { - String[] orSeparated = cleanedEachFilter.split(Filter.OR_TRIMMED); - filterByLogicalOperators.get(Filter.AND_TRIMMED).add(orSeparated[0].trim()); - Arrays.stream(orSeparated) - .skip(1) - .forEach(f -> filterByLogicalOperators.get(Filter.OR_TRIMMED).add(f.trim())); - } else if (StringUtils.isNotBlank(cleanedEachFilter)){ - filterByLogicalOperators.get(Filter.AND_TRIMMED).add(cleanedEachFilter); + Map> filterByLogicalOperators = new HashMap<>(Map.of(Filter.AND_TRIMMED, new ArrayList<>(), Filter.OR_TRIMMED, new ArrayList<>())); + // (OR|AND) (m|p|r|t|d|oD|o)\. + int flag = 0; + int andIndex = filter.indexOf(Filter.AND_TRIMMED); + int orIndex = filter.indexOf(Filter.OR_TRIMMED); + while (andIndex != -1 || orIndex != -1) { + if (andIndex != -1) { + + // substring -> exclusive + int filterIndex = andIndex + 7; + boolean matches = filter.substring(andIndex, filterIndex).matches(Pattern.compile("(AND) (m|p|r|t|d|oD|o)\\.[a-z]").pattern()); + // AND m.red = '5' OR k.kkk = '5' + + int orPrecedeIndex = filter.indexOf("OR", andIndex + 3); // pirveli OR + while (orPrecedeIndex != -1 && !filter.substring(orPrecedeIndex, orPrecedeIndex + 6).matches(Pattern.compile("(OR) (m|p|r|t|d|oD|o)\\.[a-z]").pattern())) { + orPrecedeIndex = filter.indexOf("OR", orPrecedeIndex + 3); + } + + int andPrecedeIndex = filter.indexOf("AND", andIndex + 3); // shemdegi AND + while (andPrecedeIndex != -1 && !filter.substring(andPrecedeIndex, andPrecedeIndex + 7).matches(Pattern.compile("(AND) (m|p|r|t|d|oD|o)\\.[a-z]").pattern())) { + andPrecedeIndex = filter.indexOf("AND", andPrecedeIndex + 3); + } + + if (orPrecedeIndex < andPrecedeIndex && matches) { + filterByLogicalOperators.get("AND").add(filter.substring(andIndex + 3, orPrecedeIndex == -1 ? andPrecedeIndex : orPrecedeIndex).trim()); +// orIndex = orPrecedeIndex; + andIndex = andPrecedeIndex; + } + else if (andPrecedeIndex < orPrecedeIndex && matches) { + filterByLogicalOperators.get("AND").add(filter.substring(andIndex + 3, andPrecedeIndex == -1 ? orPrecedeIndex : andPrecedeIndex).trim()); + andIndex = andPrecedeIndex; +// orIndex = orPrecedeIndex; + } else { + filterByLogicalOperators.get("AND").add(filter.substring(andIndex + 3).trim()); + andIndex = andPrecedeIndex; + } + } else { + int filterIndex = orIndex + 6; + boolean matches = filter.substring(orIndex, filterIndex).matches(Pattern.compile("(OR) (m|p|r|t|d|oD|o)\\.[a-z]").pattern()); + int orPrecedeIndex = filter.indexOf("OR", orIndex + 3); + while (orPrecedeIndex != -1 && !filter.substring(orPrecedeIndex, orPrecedeIndex + 6).matches(Pattern.compile("(OR) (m|p|r|t|d|oD|o)\\.[a-z]").pattern())) { + orPrecedeIndex = filter.indexOf("OR", orPrecedeIndex + 3); + } + int andPrecedeIndex = filter.indexOf("AND", orIndex + 3); + while (andPrecedeIndex != -1 && !filter.substring(andPrecedeIndex, andPrecedeIndex + 7).matches(Pattern.compile("(AND) (m|p|r|t|d|oD|o)\\.[a-z]").pattern())) { + andPrecedeIndex = filter.indexOf("AND", andPrecedeIndex + 3); + } + if (orPrecedeIndex < andPrecedeIndex && matches) { + filterByLogicalOperators.get("OR").add(filter.substring(orIndex + 3, orPrecedeIndex == -1 ? andPrecedeIndex : orPrecedeIndex).trim()); + orIndex = orPrecedeIndex; +// andIndex = andPrecedeIndex; + } + else if (andPrecedeIndex < orPrecedeIndex && matches) { + filterByLogicalOperators.get("OR").add(filter.substring(orIndex + 3, andPrecedeIndex == -1 ? orPrecedeIndex : andPrecedeIndex).trim()); + orIndex = orPrecedeIndex; +// andIndex = andPrecedeIndex; + } else { + filterByLogicalOperators.get("OR").add(filter.substring(orIndex + 3).trim()); + orIndex = orPrecedeIndex; + } } + } + +// +// String[] andSeparated = filter.split(Filter.AND_TRIMMED); +// +// for (String eachFilter : andSeparated) { +// String cleanedEachFilter = cleanUpData(eachFilter).trim(); +// if (cleanedEachFilter.startsWith(Filter.OR_TRIMMED) && !cleanedEachFilter.startsWith(Filter.OPEN_PARENTHESIS) && !cleanedEachFilter.endsWith(Filter.CLOSE_PARENTHESIS)) { +// String[] orSeparated = cleanedEachFilter.split(Filter.OR_TRIMMED); +// filterByLogicalOperators.get(Filter.AND_TRIMMED).add(orSeparated[0].trim()); +// Arrays.stream(orSeparated) +// .skip(1) +// .forEach(f -> filterByLogicalOperators.get(Filter.OR_TRIMMED).add(f.trim())); +// } else if (StringUtils.isNotBlank(cleanedEachFilter)){ +// filterByLogicalOperators.get(Filter.AND_TRIMMED).add(cleanedEachFilter); +// } +// } return filterByLogicalOperators; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java index 788c46cc4..fdd59900f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java @@ -3,9 +3,7 @@ import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.Filter; -import java.util.ArrayList; import java.util.Arrays; -import java.util.List; import java.util.stream.Collectors; public enum Operator { diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java index ddf9af069..be787a7f0 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java @@ -3,6 +3,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.regex.Pattern; import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.index.query.AbstractQueryBuilder; @@ -28,15 +29,23 @@ public void setUp() { @Test public void parseFiltersByLogicalOperators() { - String filter = "AND m.medicalRecordId = '15' OR m.medicalRecordSomething LIKE '55555' OR m.medicalRecordSomethingg = '55552' AND m.dynamicFields.ragac = '55' OR m.medicalRecordName = '213'"; + String filter = "AND m.medicalRecordId = '15' " + + "OR m.medicalRecordSomething LIKE '55555' " + + "OR m.medicalRecordSomethingg = '55552' " + + "AND t.tissueRecord IS NOT NULL " + + "AND m.dynamicFields.ragac = '55' " + + "OR m.medicalRecordName = '213' " + + "OR m.mrNotes = 'MEDICAL_RECORD_NOTESS' " + + "AND m.medicalMedical = 'something AND something' " + + "AND ( oD.request = 'review' OR oD.request = 'no' )"; collectionQueryBuilder.setFilter(filter); Map> parsedFilters = collectionQueryBuilder.parseFiltersByLogicalOperators(); for (Map.Entry> eachFilter: parsedFilters.entrySet()) { if (eachFilter.getKey().equals("AND")) { - Assert.assertArrayEquals(new ArrayList<>(List.of("m.medicalRecordId = '15'", "m.dynamicFields.ragac = '55'")).toArray(), + Assert.assertArrayEquals(new ArrayList<>(List.of("m.medicalRecordId = '15'", "m.dynamicFields.ragac = '55'", "m.medicalMedical = 'something AND something'", "t.tissueRecord IS NOT NULL", "( oD.request = 'review' OR oD.request = 'no' )")).toArray(), eachFilter.getValue().toArray()); } else { - Assert.assertArrayEquals(new ArrayList<>(List.of("m.medicalRecordSomething LIKE '55555'", "m.medicalRecordSomethingg = '55552'", "m.medicalRecordName = '213'")).toArray(), eachFilter.getValue().toArray()); + Assert.assertArrayEquals(new ArrayList<>(List.of("m.medicalRecordSomething LIKE '55555'", "m.medicalRecordSomethingg = '55552'", "m.medicalRecordName = '213'", "m.mrNotes = 'MEDICAL_RECORD_NOTESS'")).toArray(), eachFilter.getValue().toArray()); } } } From f42d7e3bcef230a8a1fa9b38d60f074379acf911 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 24 Dec 2021 17:46:27 +0400 Subject: [PATCH 258/405] DDP-7121 | change logic of parseFiltersByLogicalOperators to cover all edge cases --- .../filter/DsmAbstractQueryBuilder.java | 106 ++++++++++-------- .../filter/CollectionQueryBuilderTest.java | 20 +++- 2 files changed, 76 insertions(+), 50 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java index 9521da0cf..c0ebd08d7 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java @@ -12,11 +12,19 @@ import org.elasticsearch.index.query.QueryBuilder; import java.util.*; +import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Pattern; +import java.util.stream.Collectors; public abstract class DsmAbstractQueryBuilder { protected static final String DSM_WITH_DOT = ESObjectConstants.DSM + DBConstants.ALIAS_DELIMITER; + public static final String DSM_ALIAS_REGEX = "(m|p|r|t|d|oD|o|k|JS|\\()(\\.|\\s)*([a-z]|O)"; + public static final String OR_DSM_ALIAS_REGEX = "(OR) " + DSM_ALIAS_REGEX; + public static final String AND_DSM_ALIAS_REGEX = "(AND) " + DSM_ALIAS_REGEX; + public static final int AND_PATTERN_MATCHER_NUMBER = 7; + public static final int OR_PATTERN_MATCHER_NUMBER = 6; + public static final int MINIMUM_STEP_FROM_OPERATOR = 3; protected String filter; protected Parser parser; protected BoolQueryBuilder boolQueryBuilder; @@ -78,88 +86,94 @@ protected String buildPath() { protected abstract void buildEachQuery(FilterStrategy filterStrategy); protected Map> parseFiltersByLogicalOperators() { - Map> filterByLogicalOperators = new HashMap<>(Map.of(Filter.AND_TRIMMED, new ArrayList<>(), Filter.OR_TRIMMED, new ArrayList<>())); - // (OR|AND) (m|p|r|t|d|oD|o)\. - int flag = 0; + Map> filterByLogicalOperators = new ConcurrentHashMap<>(Map.of(Filter.AND_TRIMMED, new ArrayList<>(), Filter.OR_TRIMMED, + new ArrayList<>())); int andIndex = filter.indexOf(Filter.AND_TRIMMED); int orIndex = filter.indexOf(Filter.OR_TRIMMED); while (andIndex != -1 || orIndex != -1) { + andIndex = findProperOperatorSplitterIndex(Filter.AND_TRIMMED, andIndex, AND_PATTERN_MATCHER_NUMBER, MINIMUM_STEP_FROM_OPERATOR); + orIndex = findProperOperatorSplitterIndex(Filter.OR_TRIMMED, orIndex, OR_PATTERN_MATCHER_NUMBER, MINIMUM_STEP_FROM_OPERATOR); if (andIndex != -1) { - // substring -> exclusive - int filterIndex = andIndex + 7; - boolean matches = filter.substring(andIndex, filterIndex).matches(Pattern.compile("(AND) (m|p|r|t|d|oD|o)\\.[a-z]").pattern()); - // AND m.red = '5' OR k.kkk = '5' + int filterIndex = andIndex + AND_PATTERN_MATCHER_NUMBER; + boolean matches = + filter.substring(andIndex, filterIndex).matches(Pattern.compile(AND_DSM_ALIAS_REGEX).pattern()); - int orPrecedeIndex = filter.indexOf("OR", andIndex + 3); // pirveli OR - while (orPrecedeIndex != -1 && !filter.substring(orPrecedeIndex, orPrecedeIndex + 6).matches(Pattern.compile("(OR) (m|p|r|t|d|oD|o)\\.[a-z]").pattern())) { - orPrecedeIndex = filter.indexOf("OR", orPrecedeIndex + 3); + int orPrecedeIndex = filter.indexOf(Filter.OR_TRIMMED, andIndex + MINIMUM_STEP_FROM_OPERATOR); + while (orPrecedeIndex != -1 && !filter.substring(orPrecedeIndex, orPrecedeIndex + OR_PATTERN_MATCHER_NUMBER).matches(Pattern.compile( + OR_DSM_ALIAS_REGEX).pattern())) { + orPrecedeIndex = filter.indexOf(Filter.OR_TRIMMED, orPrecedeIndex + MINIMUM_STEP_FROM_OPERATOR); } - int andPrecedeIndex = filter.indexOf("AND", andIndex + 3); // shemdegi AND - while (andPrecedeIndex != -1 && !filter.substring(andPrecedeIndex, andPrecedeIndex + 7).matches(Pattern.compile("(AND) (m|p|r|t|d|oD|o)\\.[a-z]").pattern())) { - andPrecedeIndex = filter.indexOf("AND", andPrecedeIndex + 3); + int andPrecedeIndex = filter.indexOf(Filter.AND_TRIMMED, andIndex + MINIMUM_STEP_FROM_OPERATOR); + while (andPrecedeIndex != -1 && !filter.substring(andPrecedeIndex, andPrecedeIndex + AND_PATTERN_MATCHER_NUMBER).matches(Pattern.compile( + AND_DSM_ALIAS_REGEX).pattern())) { + andPrecedeIndex = filter.indexOf(Filter.AND_TRIMMED, andPrecedeIndex + MINIMUM_STEP_FROM_OPERATOR); } if (orPrecedeIndex < andPrecedeIndex && matches) { - filterByLogicalOperators.get("AND").add(filter.substring(andIndex + 3, orPrecedeIndex == -1 ? andPrecedeIndex : orPrecedeIndex).trim()); -// orIndex = orPrecedeIndex; + filterByLogicalOperators.get(Filter.AND_TRIMMED).add(filter.substring(andIndex + MINIMUM_STEP_FROM_OPERATOR, orPrecedeIndex == -1 ? andPrecedeIndex : orPrecedeIndex).trim()); andIndex = andPrecedeIndex; } else if (andPrecedeIndex < orPrecedeIndex && matches) { - filterByLogicalOperators.get("AND").add(filter.substring(andIndex + 3, andPrecedeIndex == -1 ? orPrecedeIndex : andPrecedeIndex).trim()); + filterByLogicalOperators.get(Filter.AND_TRIMMED).add(filter.substring(andIndex + MINIMUM_STEP_FROM_OPERATOR, andPrecedeIndex == -1 ? orPrecedeIndex : andPrecedeIndex).trim()); andIndex = andPrecedeIndex; -// orIndex = orPrecedeIndex; } else { - filterByLogicalOperators.get("AND").add(filter.substring(andIndex + 3).trim()); + filterByLogicalOperators.get(Filter.AND_TRIMMED).add(filter.substring(andIndex + MINIMUM_STEP_FROM_OPERATOR).trim()); andIndex = andPrecedeIndex; } } else { - int filterIndex = orIndex + 6; - boolean matches = filter.substring(orIndex, filterIndex).matches(Pattern.compile("(OR) (m|p|r|t|d|oD|o)\\.[a-z]").pattern()); - int orPrecedeIndex = filter.indexOf("OR", orIndex + 3); - while (orPrecedeIndex != -1 && !filter.substring(orPrecedeIndex, orPrecedeIndex + 6).matches(Pattern.compile("(OR) (m|p|r|t|d|oD|o)\\.[a-z]").pattern())) { - orPrecedeIndex = filter.indexOf("OR", orPrecedeIndex + 3); + int filterIndex = orIndex + OR_PATTERN_MATCHER_NUMBER; + boolean matches = filter.substring(orIndex, filterIndex).matches(Pattern.compile(OR_DSM_ALIAS_REGEX).pattern()); + int orPrecedeIndex = filter.indexOf(Filter.OR_TRIMMED, orIndex + MINIMUM_STEP_FROM_OPERATOR); + while (orPrecedeIndex != -1 && !filter.substring(orPrecedeIndex, orPrecedeIndex + OR_PATTERN_MATCHER_NUMBER).matches(Pattern.compile( + OR_DSM_ALIAS_REGEX).pattern())) { + orPrecedeIndex = filter.indexOf(Filter.OR_TRIMMED, orPrecedeIndex + MINIMUM_STEP_FROM_OPERATOR); } - int andPrecedeIndex = filter.indexOf("AND", orIndex + 3); - while (andPrecedeIndex != -1 && !filter.substring(andPrecedeIndex, andPrecedeIndex + 7).matches(Pattern.compile("(AND) (m|p|r|t|d|oD|o)\\.[a-z]").pattern())) { - andPrecedeIndex = filter.indexOf("AND", andPrecedeIndex + 3); + int andPrecedeIndex = filter.indexOf(Filter.AND_TRIMMED, orIndex + MINIMUM_STEP_FROM_OPERATOR); + while (andPrecedeIndex != -1 && !filter.substring(andPrecedeIndex, andPrecedeIndex + AND_PATTERN_MATCHER_NUMBER).matches(Pattern.compile( + AND_DSM_ALIAS_REGEX).pattern())) { + andPrecedeIndex = filter.indexOf(Filter.AND_TRIMMED, andPrecedeIndex + MINIMUM_STEP_FROM_OPERATOR); } if (orPrecedeIndex < andPrecedeIndex && matches) { - filterByLogicalOperators.get("OR").add(filter.substring(orIndex + 3, orPrecedeIndex == -1 ? andPrecedeIndex : orPrecedeIndex).trim()); + filterByLogicalOperators.get(Filter.OR_TRIMMED).add(filter.substring(orIndex + MINIMUM_STEP_FROM_OPERATOR, orPrecedeIndex == -1 ? andPrecedeIndex : orPrecedeIndex).trim()); orIndex = orPrecedeIndex; -// andIndex = andPrecedeIndex; } else if (andPrecedeIndex < orPrecedeIndex && matches) { - filterByLogicalOperators.get("OR").add(filter.substring(orIndex + 3, andPrecedeIndex == -1 ? orPrecedeIndex : andPrecedeIndex).trim()); + filterByLogicalOperators.get(Filter.OR_TRIMMED).add(filter.substring(orIndex + MINIMUM_STEP_FROM_OPERATOR, andPrecedeIndex == -1 ? orPrecedeIndex : andPrecedeIndex).trim()); orIndex = orPrecedeIndex; -// andIndex = andPrecedeIndex; } else { - filterByLogicalOperators.get("OR").add(filter.substring(orIndex + 3).trim()); + filterByLogicalOperators.get(Filter.OR_TRIMMED).add(filter.substring(orIndex + MINIMUM_STEP_FROM_OPERATOR).trim()); orIndex = orPrecedeIndex; } } } -// -// String[] andSeparated = filter.split(Filter.AND_TRIMMED); -// -// for (String eachFilter : andSeparated) { -// String cleanedEachFilter = cleanUpData(eachFilter).trim(); -// if (cleanedEachFilter.startsWith(Filter.OR_TRIMMED) && !cleanedEachFilter.startsWith(Filter.OPEN_PARENTHESIS) && !cleanedEachFilter.endsWith(Filter.CLOSE_PARENTHESIS)) { -// String[] orSeparated = cleanedEachFilter.split(Filter.OR_TRIMMED); -// filterByLogicalOperators.get(Filter.AND_TRIMMED).add(orSeparated[0].trim()); -// Arrays.stream(orSeparated) -// .skip(1) -// .forEach(f -> filterByLogicalOperators.get(Filter.OR_TRIMMED).add(f.trim())); -// } else if (StringUtils.isNotBlank(cleanedEachFilter)){ -// filterByLogicalOperators.get(Filter.AND_TRIMMED).add(cleanedEachFilter); -// } -// } return filterByLogicalOperators; } + /** + * "AND ( oD.request = 'review' OR oD.request = 'no' ) " - filter + * + * @param operator -"AND", "OR" + * @param startIndex - index where first operator is found + * @param patternMatcherNumber - number to add start index to find out whether it matches alias regex | "AND ( o" + * @param nextOperatorFromNumber - number to add start index to find next operator index | OR oD. index of OR + * @return index of next proper operator after first operator, proper operator matches either AND_DSM_ALIAS_REGEX or OR_DSM_ALIAS_REGEX + */ + private int findProperOperatorSplitterIndex(String operator, int startIndex, int patternMatcherNumber, int nextOperatorFromNumber) { + String aliasRegex = "AND".equals(operator) ? AND_DSM_ALIAS_REGEX : OR_DSM_ALIAS_REGEX; + while (startIndex != -1 && !filter.substring(startIndex, startIndex + patternMatcherNumber).matches(aliasRegex)) { + startIndex = findNextOperatorIndex(operator, startIndex + nextOperatorFromNumber); + } + return startIndex; + } + + private int findNextOperatorIndex(String operator, int fromIndex) { + return filter.indexOf(operator, fromIndex); + } + private String cleanUpData(String filterValue) { final Pattern additionalValuesJsonToClean = Pattern.compile("([a-zA-Z]+\\.(additional_values_json IS NOT NULL))"); return filterValue.replaceAll(additionalValuesJsonToClean.pattern(), StringUtils.EMPTY); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java index be787a7f0..a3e30f307 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java @@ -37,15 +37,22 @@ public void parseFiltersByLogicalOperators() { "OR m.medicalRecordName = '213' " + "OR m.mrNotes = 'MEDICAL_RECORD_NOTESS' " + "AND m.medicalMedical = 'something AND something' " + - "AND ( oD.request = 'review' OR oD.request = 'no' )"; + "AND ( oD.request = 'review' OR oD.request = 'no' ) " + + "OR t.tissueRecord = '225' " + + "AND JSON_EXTRACT ( m.additiona`l_values_json , '$.seeingIfBugExists' )"; collectionQueryBuilder.setFilter(filter); Map> parsedFilters = collectionQueryBuilder.parseFiltersByLogicalOperators(); for (Map.Entry> eachFilter: parsedFilters.entrySet()) { if (eachFilter.getKey().equals("AND")) { - Assert.assertArrayEquals(new ArrayList<>(List.of("m.medicalRecordId = '15'", "m.dynamicFields.ragac = '55'", "m.medicalMedical = 'something AND something'", "t.tissueRecord IS NOT NULL", "( oD.request = 'review' OR oD.request = 'no' )")).toArray(), + Assert.assertArrayEquals(new ArrayList<>(List.of("m.medicalRecordId = '15'", + "t.tissueRecord IS NOT NULL" ,"m.dynamicFields.ragac = '55'", "m" + + ".medicalMedical = 'something AND something'", "( oD.request = 'review' OR oD.request = 'no' )", + "JSON_EXTRACT ( m.additiona`l_values_json , '$.seeingIfBugExists' )")).toArray(), eachFilter.getValue().toArray()); } else { - Assert.assertArrayEquals(new ArrayList<>(List.of("m.medicalRecordSomething LIKE '55555'", "m.medicalRecordSomethingg = '55552'", "m.medicalRecordName = '213'", "m.mrNotes = 'MEDICAL_RECORD_NOTESS'")).toArray(), eachFilter.getValue().toArray()); + Assert.assertArrayEquals(new ArrayList<>(List.of("m.medicalRecordSomething LIKE '55555'", "m.medicalRecordSomethingg = " + + "'55552'", "m.medicalRecordName = '213'", "m.mrNotes = 'MEDICAL_RECORD_NOTESS'", "t.tissueRecord = '225'")).toArray(), + eachFilter.getValue().toArray()); } } } @@ -167,6 +174,11 @@ public void dynamicFieldsQueryBuild() { Assert.assertEquals(expected, actual); } - + @Test + public void dsmAliasRegex() { + Pattern dsmAliasRegex = Pattern.compile("(AND) (m|p|r|t|d|oD|o|JSO|\\()(\\.|\\s|)([a-z]*)"); + String substringToMatch = "AND JSON_EXTRACT ( m.additional_values_json , '$.seeingIfBugExists' )".substring(0, 7); + Assert.assertTrue(dsmAliasRegex.matcher(substringToMatch).matches()); + } } \ No newline at end of file From 0df4eded54d239260e39883d9dba2ec43e1dccf7 Mon Sep 17 00:00:00 2001 From: Nika Date: Fri, 24 Dec 2021 18:11:35 +0400 Subject: [PATCH 259/405] DDP-7121 | create AndOrFilterSeparator which is responsible for grouping filters preceded by "OR" and "AND" --- .../elastic/filter/AndOrFilterSeparator.java | 121 ++++++++++++++++++ .../filter/DsmAbstractQueryBuilder.java | 104 +-------------- .../filter/AndOrFilterSeparatorTest.java | 45 +++++++ .../filter/CollectionQueryBuilderTest.java | 30 ----- 4 files changed, 170 insertions(+), 130 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparatorTest.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java new file mode 100644 index 000000000..7d0920a77 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java @@ -0,0 +1,121 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import org.broadinstitute.dsm.model.Filter; +import org.broadinstitute.dsm.statics.DBConstants; +import org.broadinstitute.dsm.statics.ESObjectConstants; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Pattern; + +public class AndOrFilterSeparator { + + public static final String DSM_ALIAS_REGEX = "(m|p|r|t|d|oD|o|k|JS|\\()(\\.|\\s)*([a-z]|O)"; + public static final String OR_DSM_ALIAS_REGEX = "(OR) " + DSM_ALIAS_REGEX; + public static final String AND_DSM_ALIAS_REGEX = "(AND) " + DSM_ALIAS_REGEX; + public static final int AND_PATTERN_MATCHER_NUMBER = 7; + public static final int OR_PATTERN_MATCHER_NUMBER = 6; + public static final int MINIMUM_STEP_FROM_OPERATOR = 3; + + private String filter; + + public AndOrFilterSeparator(String filter) { + this.filter = filter; + } + + public void setFilter(String filter) { + this.filter = filter; + } + + protected Map> parseFiltersByLogicalOperators() { + Map> filterByLogicalOperators = new ConcurrentHashMap<>(Map.of(Filter.AND_TRIMMED, new ArrayList<>(), Filter.OR_TRIMMED, + new ArrayList<>())); + int andIndex = filter.indexOf(Filter.AND_TRIMMED); + int orIndex = filter.indexOf(Filter.OR_TRIMMED); + while (andIndex != -1 || orIndex != -1) { + andIndex = findProperOperatorSplitterIndex(Filter.AND_TRIMMED, andIndex, AND_PATTERN_MATCHER_NUMBER, MINIMUM_STEP_FROM_OPERATOR); + orIndex = findProperOperatorSplitterIndex(Filter.OR_TRIMMED, orIndex, OR_PATTERN_MATCHER_NUMBER, MINIMUM_STEP_FROM_OPERATOR); + if (andIndex != -1) { + + int filterIndex = andIndex + AND_PATTERN_MATCHER_NUMBER; + boolean matches = + filter.substring(andIndex, filterIndex).matches(Pattern.compile(AND_DSM_ALIAS_REGEX).pattern()); + + int orPrecedeIndex = filter.indexOf(Filter.OR_TRIMMED, andIndex + MINIMUM_STEP_FROM_OPERATOR); + while (orPrecedeIndex != -1 && !filter.substring(orPrecedeIndex, orPrecedeIndex + OR_PATTERN_MATCHER_NUMBER).matches(Pattern.compile( + OR_DSM_ALIAS_REGEX).pattern())) { + orPrecedeIndex = filter.indexOf(Filter.OR_TRIMMED, orPrecedeIndex + MINIMUM_STEP_FROM_OPERATOR); + } + + int andPrecedeIndex = filter.indexOf(Filter.AND_TRIMMED, andIndex + MINIMUM_STEP_FROM_OPERATOR); + while (andPrecedeIndex != -1 && !filter.substring(andPrecedeIndex, andPrecedeIndex + AND_PATTERN_MATCHER_NUMBER).matches(Pattern.compile( + AND_DSM_ALIAS_REGEX).pattern())) { + andPrecedeIndex = filter.indexOf(Filter.AND_TRIMMED, andPrecedeIndex + MINIMUM_STEP_FROM_OPERATOR); + } + + if (orPrecedeIndex < andPrecedeIndex && matches) { + filterByLogicalOperators.get(Filter.AND_TRIMMED).add(filter.substring(andIndex + MINIMUM_STEP_FROM_OPERATOR, orPrecedeIndex == -1 ? andPrecedeIndex : orPrecedeIndex).trim()); + andIndex = andPrecedeIndex; + } + else if (andPrecedeIndex < orPrecedeIndex && matches) { + filterByLogicalOperators.get(Filter.AND_TRIMMED).add(filter.substring(andIndex + MINIMUM_STEP_FROM_OPERATOR, andPrecedeIndex == -1 ? orPrecedeIndex : andPrecedeIndex).trim()); + andIndex = andPrecedeIndex; + } else { + filterByLogicalOperators.get(Filter.AND_TRIMMED).add(filter.substring(andIndex + MINIMUM_STEP_FROM_OPERATOR).trim()); + andIndex = andPrecedeIndex; + } + } else { + int filterIndex = orIndex + OR_PATTERN_MATCHER_NUMBER; + boolean matches = filter.substring(orIndex, filterIndex).matches(Pattern.compile(OR_DSM_ALIAS_REGEX).pattern()); + int orPrecedeIndex = filter.indexOf(Filter.OR_TRIMMED, orIndex + MINIMUM_STEP_FROM_OPERATOR); + while (orPrecedeIndex != -1 && !filter.substring(orPrecedeIndex, orPrecedeIndex + OR_PATTERN_MATCHER_NUMBER).matches(Pattern.compile( + OR_DSM_ALIAS_REGEX).pattern())) { + orPrecedeIndex = filter.indexOf(Filter.OR_TRIMMED, orPrecedeIndex + MINIMUM_STEP_FROM_OPERATOR); + } + int andPrecedeIndex = filter.indexOf(Filter.AND_TRIMMED, orIndex + MINIMUM_STEP_FROM_OPERATOR); + while (andPrecedeIndex != -1 && !filter.substring(andPrecedeIndex, andPrecedeIndex + AND_PATTERN_MATCHER_NUMBER).matches(Pattern.compile( + AND_DSM_ALIAS_REGEX).pattern())) { + andPrecedeIndex = filter.indexOf(Filter.AND_TRIMMED, andPrecedeIndex + MINIMUM_STEP_FROM_OPERATOR); + } + if (orPrecedeIndex < andPrecedeIndex && matches) { + filterByLogicalOperators.get(Filter.OR_TRIMMED).add(filter.substring(orIndex + MINIMUM_STEP_FROM_OPERATOR, orPrecedeIndex == -1 ? andPrecedeIndex : orPrecedeIndex).trim()); + orIndex = orPrecedeIndex; + } + else if (andPrecedeIndex < orPrecedeIndex && matches) { + filterByLogicalOperators.get(Filter.OR_TRIMMED).add(filter.substring(orIndex + MINIMUM_STEP_FROM_OPERATOR, andPrecedeIndex == -1 ? orPrecedeIndex : andPrecedeIndex).trim()); + orIndex = orPrecedeIndex; + } else { + filterByLogicalOperators.get(Filter.OR_TRIMMED).add(filter.substring(orIndex + MINIMUM_STEP_FROM_OPERATOR).trim()); + orIndex = orPrecedeIndex; + } + } + + } + + return filterByLogicalOperators; + } + + /** + * "AND ( oD.request = 'review' OR oD.request = 'no' ) " - filter + * + * @param operator -"AND", "OR" + * @param startIndex - index where first operator is found + * @param patternMatcherNumber - number to add start index to find out whether it matches alias regex | "AND ( o" + * @param nextOperatorFromNumber - number to add start index to find next operator index | OR oD. index of OR + * @return index of next proper operator after first operator, proper operator matches either AND_DSM_ALIAS_REGEX or OR_DSM_ALIAS_REGEX + */ + private int findProperOperatorSplitterIndex(String operator, int startIndex, int patternMatcherNumber, int nextOperatorFromNumber) { + String aliasRegex = "AND".equals(operator) ? AND_DSM_ALIAS_REGEX : OR_DSM_ALIAS_REGEX; + while (startIndex != -1 && !filter.substring(startIndex, startIndex + patternMatcherNumber).matches(aliasRegex)) { + startIndex = findNextOperatorIndex(operator, startIndex + nextOperatorFromNumber); + } + return startIndex; + } + + private int findNextOperatorIndex(String operator, int fromIndex) { + return filter.indexOf(operator, fromIndex); + } + +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java index c0ebd08d7..f4d7bacb1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java @@ -19,22 +19,18 @@ public abstract class DsmAbstractQueryBuilder { protected static final String DSM_WITH_DOT = ESObjectConstants.DSM + DBConstants.ALIAS_DELIMITER; - public static final String DSM_ALIAS_REGEX = "(m|p|r|t|d|oD|o|k|JS|\\()(\\.|\\s)*([a-z]|O)"; - public static final String OR_DSM_ALIAS_REGEX = "(OR) " + DSM_ALIAS_REGEX; - public static final String AND_DSM_ALIAS_REGEX = "(AND) " + DSM_ALIAS_REGEX; - public static final int AND_PATTERN_MATCHER_NUMBER = 7; - public static final int OR_PATTERN_MATCHER_NUMBER = 6; - public static final int MINIMUM_STEP_FROM_OPERATOR = 3; protected String filter; protected Parser parser; protected BoolQueryBuilder boolQueryBuilder; protected QueryBuilder queryBuilder; protected BaseSplitter splitter; + protected AndOrFilterSeparator filterSeparator; public DsmAbstractQueryBuilder(String filter, Parser parser) { this(); this.parser = parser; this.filter = filter; + this.filterSeparator = new AndOrFilterSeparator(this.filter); } public static DsmAbstractQueryBuilder of(String alias) { @@ -49,6 +45,7 @@ public DsmAbstractQueryBuilder() { public void setFilter(String filter) { this.filter = filter; + this.filterSeparator = new AndOrFilterSeparator(filter); } public void setParser(Parser parser) { @@ -56,7 +53,7 @@ public void setParser(Parser parser) { } public AbstractQueryBuilder build() { - Map> parsedFilters = parseFiltersByLogicalOperators(); + Map> parsedFilters = filterSeparator.parseFiltersByLogicalOperators(); for (Map.Entry> parsedFilter: parsedFilters.entrySet()) { List filterValues = parsedFilter.getValue(); if (parsedFilter.getKey().equals(Filter.AND_TRIMMED)) { @@ -85,97 +82,4 @@ protected String buildPath() { protected abstract void buildEachQuery(FilterStrategy filterStrategy); - protected Map> parseFiltersByLogicalOperators() { - Map> filterByLogicalOperators = new ConcurrentHashMap<>(Map.of(Filter.AND_TRIMMED, new ArrayList<>(), Filter.OR_TRIMMED, - new ArrayList<>())); - int andIndex = filter.indexOf(Filter.AND_TRIMMED); - int orIndex = filter.indexOf(Filter.OR_TRIMMED); - while (andIndex != -1 || orIndex != -1) { - andIndex = findProperOperatorSplitterIndex(Filter.AND_TRIMMED, andIndex, AND_PATTERN_MATCHER_NUMBER, MINIMUM_STEP_FROM_OPERATOR); - orIndex = findProperOperatorSplitterIndex(Filter.OR_TRIMMED, orIndex, OR_PATTERN_MATCHER_NUMBER, MINIMUM_STEP_FROM_OPERATOR); - if (andIndex != -1) { - - int filterIndex = andIndex + AND_PATTERN_MATCHER_NUMBER; - boolean matches = - filter.substring(andIndex, filterIndex).matches(Pattern.compile(AND_DSM_ALIAS_REGEX).pattern()); - - int orPrecedeIndex = filter.indexOf(Filter.OR_TRIMMED, andIndex + MINIMUM_STEP_FROM_OPERATOR); - while (orPrecedeIndex != -1 && !filter.substring(orPrecedeIndex, orPrecedeIndex + OR_PATTERN_MATCHER_NUMBER).matches(Pattern.compile( - OR_DSM_ALIAS_REGEX).pattern())) { - orPrecedeIndex = filter.indexOf(Filter.OR_TRIMMED, orPrecedeIndex + MINIMUM_STEP_FROM_OPERATOR); - } - - int andPrecedeIndex = filter.indexOf(Filter.AND_TRIMMED, andIndex + MINIMUM_STEP_FROM_OPERATOR); - while (andPrecedeIndex != -1 && !filter.substring(andPrecedeIndex, andPrecedeIndex + AND_PATTERN_MATCHER_NUMBER).matches(Pattern.compile( - AND_DSM_ALIAS_REGEX).pattern())) { - andPrecedeIndex = filter.indexOf(Filter.AND_TRIMMED, andPrecedeIndex + MINIMUM_STEP_FROM_OPERATOR); - } - - if (orPrecedeIndex < andPrecedeIndex && matches) { - filterByLogicalOperators.get(Filter.AND_TRIMMED).add(filter.substring(andIndex + MINIMUM_STEP_FROM_OPERATOR, orPrecedeIndex == -1 ? andPrecedeIndex : orPrecedeIndex).trim()); - andIndex = andPrecedeIndex; - } - else if (andPrecedeIndex < orPrecedeIndex && matches) { - filterByLogicalOperators.get(Filter.AND_TRIMMED).add(filter.substring(andIndex + MINIMUM_STEP_FROM_OPERATOR, andPrecedeIndex == -1 ? orPrecedeIndex : andPrecedeIndex).trim()); - andIndex = andPrecedeIndex; - } else { - filterByLogicalOperators.get(Filter.AND_TRIMMED).add(filter.substring(andIndex + MINIMUM_STEP_FROM_OPERATOR).trim()); - andIndex = andPrecedeIndex; - } - } else { - int filterIndex = orIndex + OR_PATTERN_MATCHER_NUMBER; - boolean matches = filter.substring(orIndex, filterIndex).matches(Pattern.compile(OR_DSM_ALIAS_REGEX).pattern()); - int orPrecedeIndex = filter.indexOf(Filter.OR_TRIMMED, orIndex + MINIMUM_STEP_FROM_OPERATOR); - while (orPrecedeIndex != -1 && !filter.substring(orPrecedeIndex, orPrecedeIndex + OR_PATTERN_MATCHER_NUMBER).matches(Pattern.compile( - OR_DSM_ALIAS_REGEX).pattern())) { - orPrecedeIndex = filter.indexOf(Filter.OR_TRIMMED, orPrecedeIndex + MINIMUM_STEP_FROM_OPERATOR); - } - int andPrecedeIndex = filter.indexOf(Filter.AND_TRIMMED, orIndex + MINIMUM_STEP_FROM_OPERATOR); - while (andPrecedeIndex != -1 && !filter.substring(andPrecedeIndex, andPrecedeIndex + AND_PATTERN_MATCHER_NUMBER).matches(Pattern.compile( - AND_DSM_ALIAS_REGEX).pattern())) { - andPrecedeIndex = filter.indexOf(Filter.AND_TRIMMED, andPrecedeIndex + MINIMUM_STEP_FROM_OPERATOR); - } - if (orPrecedeIndex < andPrecedeIndex && matches) { - filterByLogicalOperators.get(Filter.OR_TRIMMED).add(filter.substring(orIndex + MINIMUM_STEP_FROM_OPERATOR, orPrecedeIndex == -1 ? andPrecedeIndex : orPrecedeIndex).trim()); - orIndex = orPrecedeIndex; - } - else if (andPrecedeIndex < orPrecedeIndex && matches) { - filterByLogicalOperators.get(Filter.OR_TRIMMED).add(filter.substring(orIndex + MINIMUM_STEP_FROM_OPERATOR, andPrecedeIndex == -1 ? orPrecedeIndex : andPrecedeIndex).trim()); - orIndex = orPrecedeIndex; - } else { - filterByLogicalOperators.get(Filter.OR_TRIMMED).add(filter.substring(orIndex + MINIMUM_STEP_FROM_OPERATOR).trim()); - orIndex = orPrecedeIndex; - } - } - - } - - return filterByLogicalOperators; - } - - /** - * "AND ( oD.request = 'review' OR oD.request = 'no' ) " - filter - * - * @param operator -"AND", "OR" - * @param startIndex - index where first operator is found - * @param patternMatcherNumber - number to add start index to find out whether it matches alias regex | "AND ( o" - * @param nextOperatorFromNumber - number to add start index to find next operator index | OR oD. index of OR - * @return index of next proper operator after first operator, proper operator matches either AND_DSM_ALIAS_REGEX or OR_DSM_ALIAS_REGEX - */ - private int findProperOperatorSplitterIndex(String operator, int startIndex, int patternMatcherNumber, int nextOperatorFromNumber) { - String aliasRegex = "AND".equals(operator) ? AND_DSM_ALIAS_REGEX : OR_DSM_ALIAS_REGEX; - while (startIndex != -1 && !filter.substring(startIndex, startIndex + patternMatcherNumber).matches(aliasRegex)) { - startIndex = findNextOperatorIndex(operator, startIndex + nextOperatorFromNumber); - } - return startIndex; - } - - private int findNextOperatorIndex(String operator, int fromIndex) { - return filter.indexOf(operator, fromIndex); - } - - private String cleanUpData(String filterValue) { - final Pattern additionalValuesJsonToClean = Pattern.compile("([a-zA-Z]+\\.(additional_values_json IS NOT NULL))"); - return filterValue.replaceAll(additionalValuesJsonToClean.pattern(), StringUtils.EMPTY); - } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparatorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparatorTest.java new file mode 100644 index 000000000..d1713d8fe --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparatorTest.java @@ -0,0 +1,45 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.*; + +public class AndOrFilterSeparatorTest { + + @Test + public void parseFiltersByLogicalOperators() { + + String filter = "AND m.medicalRecordId = '15' " + + "OR m.medicalRecordSomething LIKE '55555' " + + "OR m.medicalRecordSomethingg = '55552' " + + "AND t.tissueRecord IS NOT NULL " + + "AND m.dynamicFields.ragac = '55' " + + "OR m.medicalRecordName = '213' " + + "OR m.mrNotes = 'MEDICAL_RECORD_NOTESS' " + + "AND m.medicalMedical = 'something AND something' " + + "AND ( oD.request = 'review' OR oD.request = 'no' ) " + + "OR t.tissueRecord = '225' " + + "AND JSON_EXTRACT ( m.additiona`l_values_json , '$.seeingIfBugExists' )"; + AndOrFilterSeparator andOrFilterSeparator = new AndOrFilterSeparator(filter); + Map> parsedFilters = andOrFilterSeparator.parseFiltersByLogicalOperators(); + for (Map.Entry> eachFilter: parsedFilters.entrySet()) { + if (eachFilter.getKey().equals("AND")) { + Assert.assertArrayEquals(new ArrayList<>(List.of("m.medicalRecordId = '15'", + "t.tissueRecord IS NOT NULL" ,"m.dynamicFields.ragac = '55'", "m" + + ".medicalMedical = 'something AND something'", "( oD.request = 'review' OR oD.request = 'no' )", + "JSON_EXTRACT ( m.additiona`l_values_json , '$.seeingIfBugExists' )")).toArray(), + eachFilter.getValue().toArray()); + } else { + Assert.assertArrayEquals(new ArrayList<>(List.of("m.medicalRecordSomething LIKE '55555'", "m.medicalRecordSomethingg = " + + "'55552'", "m.medicalRecordName = '213'", "m.mrNotes = 'MEDICAL_RECORD_NOTESS'", "t.tissueRecord = '225'")).toArray(), + eachFilter.getValue().toArray()); + } + } + } +} \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java index a3e30f307..9ba6615b4 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java @@ -27,36 +27,6 @@ public void setUp() { collectionQueryBuilder.setParser(new FilterParser()); } - @Test - public void parseFiltersByLogicalOperators() { - String filter = "AND m.medicalRecordId = '15' " + - "OR m.medicalRecordSomething LIKE '55555' " + - "OR m.medicalRecordSomethingg = '55552' " + - "AND t.tissueRecord IS NOT NULL " + - "AND m.dynamicFields.ragac = '55' " + - "OR m.medicalRecordName = '213' " + - "OR m.mrNotes = 'MEDICAL_RECORD_NOTESS' " + - "AND m.medicalMedical = 'something AND something' " + - "AND ( oD.request = 'review' OR oD.request = 'no' ) " + - "OR t.tissueRecord = '225' " + - "AND JSON_EXTRACT ( m.additiona`l_values_json , '$.seeingIfBugExists' )"; - collectionQueryBuilder.setFilter(filter); - Map> parsedFilters = collectionQueryBuilder.parseFiltersByLogicalOperators(); - for (Map.Entry> eachFilter: parsedFilters.entrySet()) { - if (eachFilter.getKey().equals("AND")) { - Assert.assertArrayEquals(new ArrayList<>(List.of("m.medicalRecordId = '15'", - "t.tissueRecord IS NOT NULL" ,"m.dynamicFields.ragac = '55'", "m" + - ".medicalMedical = 'something AND something'", "( oD.request = 'review' OR oD.request = 'no' )", - "JSON_EXTRACT ( m.additiona`l_values_json , '$.seeingIfBugExists' )")).toArray(), - eachFilter.getValue().toArray()); - } else { - Assert.assertArrayEquals(new ArrayList<>(List.of("m.medicalRecordSomething LIKE '55555'", "m.medicalRecordSomethingg = " + - "'55552'", "m.medicalRecordName = '213'", "m.mrNotes = 'MEDICAL_RECORD_NOTESS'", "t.tissueRecord = '225'")).toArray(), - eachFilter.getValue().toArray()); - } - } - } - @Test public void collectionBuild() { From 5cafbd093d95048d58314574f0651a85e06238e1 Mon Sep 17 00:00:00 2001 From: Nika Date: Fri, 24 Dec 2021 19:07:04 +0400 Subject: [PATCH 260/405] DDP-7121 | refactor AndOrFilterSeparator, remove code duplication, extract methods --- .../elastic/filter/AndOrFilterSeparator.java | 105 ++++++++---------- .../filter/DsmAbstractQueryBuilder.java | 8 +- 2 files changed, 51 insertions(+), 62 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java index 7d0920a77..5112f2a27 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java @@ -1,8 +1,6 @@ package org.broadinstitute.dsm.model.elastic.filter; import org.broadinstitute.dsm.model.Filter; -import org.broadinstitute.dsm.statics.DBConstants; -import org.broadinstitute.dsm.statics.ESObjectConstants; import java.util.ArrayList; import java.util.List; @@ -38,62 +36,11 @@ protected Map> parseFiltersByLogicalOperators() { andIndex = findProperOperatorSplitterIndex(Filter.AND_TRIMMED, andIndex, AND_PATTERN_MATCHER_NUMBER, MINIMUM_STEP_FROM_OPERATOR); orIndex = findProperOperatorSplitterIndex(Filter.OR_TRIMMED, orIndex, OR_PATTERN_MATCHER_NUMBER, MINIMUM_STEP_FROM_OPERATOR); if (andIndex != -1) { - - int filterIndex = andIndex + AND_PATTERN_MATCHER_NUMBER; - boolean matches = - filter.substring(andIndex, filterIndex).matches(Pattern.compile(AND_DSM_ALIAS_REGEX).pattern()); - - int orPrecedeIndex = filter.indexOf(Filter.OR_TRIMMED, andIndex + MINIMUM_STEP_FROM_OPERATOR); - while (orPrecedeIndex != -1 && !filter.substring(orPrecedeIndex, orPrecedeIndex + OR_PATTERN_MATCHER_NUMBER).matches(Pattern.compile( - OR_DSM_ALIAS_REGEX).pattern())) { - orPrecedeIndex = filter.indexOf(Filter.OR_TRIMMED, orPrecedeIndex + MINIMUM_STEP_FROM_OPERATOR); - } - - int andPrecedeIndex = filter.indexOf(Filter.AND_TRIMMED, andIndex + MINIMUM_STEP_FROM_OPERATOR); - while (andPrecedeIndex != -1 && !filter.substring(andPrecedeIndex, andPrecedeIndex + AND_PATTERN_MATCHER_NUMBER).matches(Pattern.compile( - AND_DSM_ALIAS_REGEX).pattern())) { - andPrecedeIndex = filter.indexOf(Filter.AND_TRIMMED, andPrecedeIndex + MINIMUM_STEP_FROM_OPERATOR); - } - - if (orPrecedeIndex < andPrecedeIndex && matches) { - filterByLogicalOperators.get(Filter.AND_TRIMMED).add(filter.substring(andIndex + MINIMUM_STEP_FROM_OPERATOR, orPrecedeIndex == -1 ? andPrecedeIndex : orPrecedeIndex).trim()); - andIndex = andPrecedeIndex; - } - else if (andPrecedeIndex < orPrecedeIndex && matches) { - filterByLogicalOperators.get(Filter.AND_TRIMMED).add(filter.substring(andIndex + MINIMUM_STEP_FROM_OPERATOR, andPrecedeIndex == -1 ? orPrecedeIndex : andPrecedeIndex).trim()); - andIndex = andPrecedeIndex; - } else { - filterByLogicalOperators.get(Filter.AND_TRIMMED).add(filter.substring(andIndex + MINIMUM_STEP_FROM_OPERATOR).trim()); - andIndex = andPrecedeIndex; - } + andIndex = getIndex(filterByLogicalOperators, andIndex, Filter.AND_TRIMMED); } else { - int filterIndex = orIndex + OR_PATTERN_MATCHER_NUMBER; - boolean matches = filter.substring(orIndex, filterIndex).matches(Pattern.compile(OR_DSM_ALIAS_REGEX).pattern()); - int orPrecedeIndex = filter.indexOf(Filter.OR_TRIMMED, orIndex + MINIMUM_STEP_FROM_OPERATOR); - while (orPrecedeIndex != -1 && !filter.substring(orPrecedeIndex, orPrecedeIndex + OR_PATTERN_MATCHER_NUMBER).matches(Pattern.compile( - OR_DSM_ALIAS_REGEX).pattern())) { - orPrecedeIndex = filter.indexOf(Filter.OR_TRIMMED, orPrecedeIndex + MINIMUM_STEP_FROM_OPERATOR); - } - int andPrecedeIndex = filter.indexOf(Filter.AND_TRIMMED, orIndex + MINIMUM_STEP_FROM_OPERATOR); - while (andPrecedeIndex != -1 && !filter.substring(andPrecedeIndex, andPrecedeIndex + AND_PATTERN_MATCHER_NUMBER).matches(Pattern.compile( - AND_DSM_ALIAS_REGEX).pattern())) { - andPrecedeIndex = filter.indexOf(Filter.AND_TRIMMED, andPrecedeIndex + MINIMUM_STEP_FROM_OPERATOR); - } - if (orPrecedeIndex < andPrecedeIndex && matches) { - filterByLogicalOperators.get(Filter.OR_TRIMMED).add(filter.substring(orIndex + MINIMUM_STEP_FROM_OPERATOR, orPrecedeIndex == -1 ? andPrecedeIndex : orPrecedeIndex).trim()); - orIndex = orPrecedeIndex; - } - else if (andPrecedeIndex < orPrecedeIndex && matches) { - filterByLogicalOperators.get(Filter.OR_TRIMMED).add(filter.substring(orIndex + MINIMUM_STEP_FROM_OPERATOR, andPrecedeIndex == -1 ? orPrecedeIndex : andPrecedeIndex).trim()); - orIndex = orPrecedeIndex; - } else { - filterByLogicalOperators.get(Filter.OR_TRIMMED).add(filter.substring(orIndex + MINIMUM_STEP_FROM_OPERATOR).trim()); - orIndex = orPrecedeIndex; - } + orIndex = getIndex(filterByLogicalOperators, orIndex, Filter.OR_TRIMMED); } - } - return filterByLogicalOperators; } @@ -107,15 +54,61 @@ else if (andPrecedeIndex < orPrecedeIndex && matches) { * @return index of next proper operator after first operator, proper operator matches either AND_DSM_ALIAS_REGEX or OR_DSM_ALIAS_REGEX */ private int findProperOperatorSplitterIndex(String operator, int startIndex, int patternMatcherNumber, int nextOperatorFromNumber) { - String aliasRegex = "AND".equals(operator) ? AND_DSM_ALIAS_REGEX : OR_DSM_ALIAS_REGEX; + String aliasRegex = getAliasRegexByOperator(operator); while (startIndex != -1 && !filter.substring(startIndex, startIndex + patternMatcherNumber).matches(aliasRegex)) { startIndex = findNextOperatorIndex(operator, startIndex + nextOperatorFromNumber); } return startIndex; } + private String getAliasRegexByOperator(String operator) { + return "AND".equals(operator) ? AND_DSM_ALIAS_REGEX : OR_DSM_ALIAS_REGEX; + } + private int findNextOperatorIndex(String operator, int fromIndex) { return filter.indexOf(operator, fromIndex); } + private int getIndex(Map> filterByLogicalOperators, int index, String operator) { + boolean matches = filter.substring(index, getFilterIndex(index, getPatternMatcherNumberByOperator(operator))).matches(Pattern.compile(getAliasRegexByOperator(operator)).pattern()); + + int orPrecedeIndex = findNextOperatorIndex(Filter.OR_TRIMMED, index + MINIMUM_STEP_FROM_OPERATOR); + orPrecedeIndex = findProperOperatorSplitterIndex(Filter.OR_TRIMMED, orPrecedeIndex, OR_PATTERN_MATCHER_NUMBER, MINIMUM_STEP_FROM_OPERATOR); + + int andPrecedeIndex = findNextOperatorIndex(Filter.AND_TRIMMED, index + MINIMUM_STEP_FROM_OPERATOR); + andPrecedeIndex = findProperOperatorSplitterIndex(Filter.AND_TRIMMED, andPrecedeIndex, AND_PATTERN_MATCHER_NUMBER, MINIMUM_STEP_FROM_OPERATOR); + + if (isLeftMostOR(matches, orPrecedeIndex, andPrecedeIndex)) { + filterByLogicalOperators.get(operator).add(filter.substring(index + MINIMUM_STEP_FROM_OPERATOR, orPrecedeIndex == -1 ? andPrecedeIndex : orPrecedeIndex).trim()); + index = isAndOperator(operator) ? andPrecedeIndex : orPrecedeIndex; + } else if (isLeftMostAND(matches, orPrecedeIndex, andPrecedeIndex)) { + filterByLogicalOperators.get(operator).add(filter.substring(index + MINIMUM_STEP_FROM_OPERATOR, andPrecedeIndex == -1 ? orPrecedeIndex : andPrecedeIndex).trim()); + index = isAndOperator(operator) ? andPrecedeIndex : orPrecedeIndex; + } else { + filterByLogicalOperators.get(operator).add(filter.substring(index + MINIMUM_STEP_FROM_OPERATOR).trim()); + index = isAndOperator(operator) ? andPrecedeIndex : orPrecedeIndex; + } + return index; + } + + private boolean isLeftMostOR(boolean matches, int orPrecedeIndex, int andPrecedeIndex) { + return orPrecedeIndex < andPrecedeIndex && matches; + } + + private boolean isLeftMostAND(boolean matches, int orPrecedeIndex, int andPrecedeIndex) { + return andPrecedeIndex < orPrecedeIndex && matches; + } + + private boolean isAndOperator(String operator) { + return "AND".equals(operator); + } + + private int getPatternMatcherNumberByOperator(String operator) { + return isAndOperator(operator) ? AND_PATTERN_MATCHER_NUMBER : OR_PATTERN_MATCHER_NUMBER; + } + + private int getFilterIndex(int andIndex, int patternMatcherNumber) { + return andIndex + patternMatcherNumber; + } + } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java index f4d7bacb1..1c9880a18 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java @@ -1,9 +1,7 @@ package org.broadinstitute.dsm.model.elastic.filter; -import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.Filter; import org.broadinstitute.dsm.model.elastic.Util; -import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.statics.ESObjectConstants; @@ -11,10 +9,8 @@ import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.regex.Pattern; -import java.util.stream.Collectors; +import java.util.List; +import java.util.Map; public abstract class DsmAbstractQueryBuilder { From c3307c57ac45f91da5c69fb14e2894100f8f32d1 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 27 Dec 2021 12:08:35 +0400 Subject: [PATCH 261/405] DDP-7121 | fix JSON_EXTRACT bug --- .../java/org/broadinstitute/dsm/model/Filter.java | 6 ++++-- .../model/elastic/filter/AndOrFilterSeparator.java | 12 ++++++++++++ .../model/elastic/filter/JsonExtractSplitter.java | 6 ++++++ .../broadinstitute/dsm/model/elastic/UtilTest.java | 7 +++++++ .../elastic/filter/JsonExtractSplitterTest.java | 6 ++++++ 5 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/Filter.java b/src/main/java/org/broadinstitute/dsm/model/Filter.java index 1085988d7..f6d5854ad 100644 --- a/src/main/java/org/broadinstitute/dsm/model/Filter.java +++ b/src/main/java/org/broadinstitute/dsm/model/Filter.java @@ -199,7 +199,9 @@ else if (filter.isNotEmpty()) { } } else if (ADDITIONAL_VALUES.equals(filter.getType())) { - query = AND + " JSON_EXTRACT ( " + filter.getParentName() + DBConstants.ALIAS_DELIMITER + dbElement.getColumnName() + " , '$." + filter.getFilter2().getName() + "' ) "; + String jsonExtract = "JSON_EXTRACT ( "; + query = AND + + jsonExtract + filter.getParentName() + DBConstants.ALIAS_DELIMITER + dbElement.getColumnName() + " , '$." + filter.getFilter2().getName() + "' ) "; if (filter.isEmpty()) { finalQuery = query + IS_NULL + " "; } @@ -209,7 +211,7 @@ else if (filter.isNotEmpty()) { else { String notNullQuery = AND + filter.getParentName() + DBConstants.ALIAS_DELIMITER + dbElement.getColumnName() + IS_NOT_NULL; if (filter.getFilter1() != null && filter.getFilter1().getValue() != null && StringUtils.isNotBlank(String.valueOf(filter.getFilter1().getValue()))) { - query = AND + " JSON_EXTRACT ( " + filter.getParentName() + DBConstants.ALIAS_DELIMITER + dbElement.getColumnName() + " , '$." + filter.getFilter2().getName() + "' ) "; + query = AND + jsonExtract + filter.getParentName() + DBConstants.ALIAS_DELIMITER + dbElement.getColumnName() + " , '$." + filter.getFilter2().getName() + "' ) "; if (filter.isExactMatch()) { query += EQUALS + "'#'"; query = query.replaceAll("#", String.valueOf(filter.getFilter1().getValue())); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java index 5112f2a27..c56c20ab6 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java @@ -7,6 +7,7 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Pattern; +import java.util.stream.Collectors; public class AndOrFilterSeparator { @@ -41,9 +42,20 @@ protected Map> parseFiltersByLogicalOperators() { orIndex = getIndex(filterByLogicalOperators, orIndex, Filter.OR_TRIMMED); } } + handleSpecialCases(filterByLogicalOperators); return filterByLogicalOperators; } + private void handleSpecialCases(Map> filterByLogicalOperators) { + final String additionalValuesJsonIsNotNull = "additional_values_json IS NOT NULL"; + for (Map.Entry> entry: filterByLogicalOperators.entrySet()) { + List filteredByNotAdditionalValuesIsNotNull = entry.getValue().stream() + .filter(f -> !f.contains(additionalValuesJsonIsNotNull)) + .collect(Collectors.toList()); + filterByLogicalOperators.put(entry.getKey(), filteredByNotAdditionalValuesIsNotNull); + } + } + /** * "AND ( oD.request = 'review' OR oD.request = 'no' ) " - filter * diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitter.java index c13d666bc..8ab2ad672 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitter.java @@ -24,6 +24,12 @@ public String[] split() { return decoratedSplitter.split(); } + @Override + public String getInnerProperty() { + String[] separatedByDot = getFieldWithAlias()[1].split(ElasticSearchUtil.DOT_SEPARATOR); + return String.join(".", separatedByDot[0], Util.underscoresToCamelCase(separatedByDot[1])); + } + @Override protected String[] getFieldWithAlias() { String[] splittedByJsonExtractAndComma = splittedFilter[0] diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java index 167888e38..d842fb22d 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java @@ -122,4 +122,11 @@ public void camelCaseToPascalSnakeCase() { assertEquals("TEST", pascalSnakeCase2); } + @Test + public void camelCaseIfContainsDot() { + String field = "dynamicFields.Scooby"; + String converted = Util.underscoresToCamelCase(field); + assertEquals("dynamicFields.scooby", converted); + } + } \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitterTest.java index 20ae479f2..98c626b1a 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitterTest.java @@ -28,6 +28,12 @@ public void getInnerProperty() { Assert.assertEquals("dynamicFields.seeingIfBugExists", splitter.getInnerProperty()); } + @Test + public void getInnerPropertyIfUpperCaseAfterDot() { + splitter.setFilter("JSON_EXTRACT ( m.additional_values_json , '$.Scooby' ) = 'true'"); + Assert.assertEquals("dynamicFields.scooby", splitter.getInnerProperty()); + } + @Test public void getIsNotNullValue() { filter = "JSON_EXTRACT ( m.additional_values_json , '$.seeingIfBugExists' ) IS NOT NULL"; From 5544573aa85fcdbcdd34c86b5ac82453e9a2e0ab Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 27 Dec 2021 15:11:25 +0400 Subject: [PATCH 262/405] DDP-7121 | change method signature for getParticiupantsByRangeAndFilter() based on new design --- .../elastic/filter/AndOrFilterSeparator.java | 2 +- .../model/elastic/search/ElasticSearch.java | 5 +- .../elastic/search/ElasticSearchable.java | 4 +- .../model/participant/ParticipantWrapper.java | 93 ++++++++++--------- .../participant/ParticipantWrapperTest.java | 3 +- 5 files changed, 58 insertions(+), 49 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java index c56c20ab6..75122d28e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java @@ -38,7 +38,7 @@ protected Map> parseFiltersByLogicalOperators() { orIndex = findProperOperatorSplitterIndex(Filter.OR_TRIMMED, orIndex, OR_PATTERN_MATCHER_NUMBER, MINIMUM_STEP_FROM_OPERATOR); if (andIndex != -1) { andIndex = getIndex(filterByLogicalOperators, andIndex, Filter.AND_TRIMMED); - } else { + } else if (orIndex != -1){ orIndex = getIndex(filterByLogicalOperators, orIndex, Filter.OR_TRIMMED); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java index adb9f8ebf..2b3afcd0c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java @@ -153,7 +153,7 @@ public long getParticipantsSize(String esParticipantsIndex) { } @Override - public ElasticSearch getParticipantsByRangeAndFilter(String esParticipantsIndex, int from, int to, String filter) { + public ElasticSearch getParticipantsByRangeAndFilter(String esParticipantsIndex, int from, int to, AbstractQueryBuilder queryBuilder) { if (to <= 0) throw new IllegalArgumentException("incorrect from/to range"); logger.info("Collecting ES data"); SearchResponse response; @@ -161,8 +161,7 @@ public ElasticSearch getParticipantsByRangeAndFilter(String esParticipantsIndex, int scrollSize = to - from; SearchRequest searchRequest = new SearchRequest(Objects.requireNonNull(esParticipantsIndex)); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - AbstractQueryBuilder> esQuery = ElasticSearchUtil.createESQuery(filter); - searchSourceBuilder.query(esQuery).sort(ElasticSearchUtil.PROFILE_CREATED_AT, SortOrder.ASC); + searchSourceBuilder.query(queryBuilder).sort(ElasticSearchUtil.PROFILE_CREATED_AT, SortOrder.ASC); searchSourceBuilder.size(scrollSize); searchSourceBuilder.from(from); searchRequest.source(searchSourceBuilder); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchable.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchable.java index ff7c09e2f..fb09f45d7 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchable.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchable.java @@ -1,5 +1,7 @@ package org.broadinstitute.dsm.model.elastic.search; +import org.elasticsearch.index.query.AbstractQueryBuilder; + import java.util.List; public interface ElasticSearchable { @@ -11,7 +13,7 @@ public interface ElasticSearchable { long getParticipantsSize(String esParticipantsIndex); - ElasticSearch getParticipantsByRangeAndFilter(String esParticipantsIndex, int from, int to, String filter); + ElasticSearch getParticipantsByRangeAndFilter(String esParticipantsIndex, int from, int to, AbstractQueryBuilder queryBuilder); ElasticSearch getParticipantsByRangeAndIds(String participantIndexES, int from, int to, List participantIds); diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java index b1da40d4e..2f1e168ab 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java @@ -133,56 +133,63 @@ else if (DBConstants.DDP_PARTICIPANT_DATA_ALIAS.equals(source)) { // participantIdsToFetch = new ArrayList<>(abstractionActivities.keySet()); // } else if ("ES".equals(source)){ //source is not of any study-manager table so it must be ES - esData = elasticSearchable.getParticipantsByRangeAndFilter(ddpInstance.getParticipantIndexES(), participantWrapperPayload.getFrom(), - participantWrapperPayload.getTo(), filters.get(source)); - participantIdsToFetch = esData.getEsParticipants().stream().map(ElasticSearchParticipantDto::getParticipantId) - .collect( - Collectors.toList()); +// esData = elasticSearchable.getParticipantsByRangeAndFilter(ddpInstance.getParticipantIndexES(), participantWrapperPayload.getFrom(), +// participantWrapperPayload.getTo(), boolQueryBuilder); +// participantIdsToFetch = esData.getEsParticipants().stream().map(ElasticSearchParticipantDto::getParticipantId) +// .collect( +// Collectors.toList()); boolQueryBuilder.must(ElasticSearchUtil.createESQuery(filters.get(source))); } } } - if (esData.getEsParticipants().isEmpty()) { - esData = elasticSearchable.getParticipantsByRangeAndIds(ddpInstance.getParticipantIndexES(), participantWrapperPayload.getFrom(), - participantWrapperPayload.getTo(), participantIdsToFetch); - } - if (participants.isEmpty()) { - participants = Participant.getParticipantsByIds(ddpInstance.getName(), participantIdsToFetch); - } - if (medicalRecords.isEmpty() && ddpInstance.isHasRole()) { - medicalRecords = MedicalRecord.getMedicalRecordsByParticipantIds(ddpInstance.getName(), participantIdsToFetch); - } - if (oncHistoryDetails.isEmpty() && ddpInstance.isHasRole()) { - oncHistoryDetails = OncHistoryDetail.getOncHistoryDetailsByParticipantIds(ddpInstance.getName(), participantIdsToFetch); - } - if (kitRequests.isEmpty() && DDPInstanceDao.getRole(ddpInstance.getName(), DBConstants.KIT_REQUEST_ACTIVATED)) { //only needed if study is shipping samples per DSM - //get only kitRequests for the filtered pts - if (Objects.nonNull(esData) && !esData.getEsParticipants().isEmpty()) { - logger.info("About to query for kits from " + esData.getEsParticipants().size() + " participants"); - kitRequests = KitRequestShipping.getKitRequestsByParticipantIds(ddpInstance, participantIdsToFetch); - } - } - if (participantData.isEmpty()) { - participantData = new ParticipantDataDao().getParticipantDataByParticipantIds(participantIdsToFetch); - //if study is AT - if ("atcp".equals(ddpInstance.getName())) { - DefaultValues defaultValues = new DefaultValues(participantData, esData.getEsParticipants(), ddpInstance, null); - participantData = defaultValues.addDefaultValues(); - } - } - if (abstractionActivities.isEmpty()) { - abstractionActivities = AbstractionActivity.getAllAbstractionActivityByRealm(ddpInstance.getName()); - } - if (abstractionSummary.isEmpty()) { - abstractionSummary = AbstractionFinal.getAbstractionFinal(ddpInstance.getName()); - } - if (proxiesByParticipantIds.isEmpty()) { - proxiesByParticipantIds = getProxiesWithParticipantIdsFromElasticList(ddpInstance.getUsersIndexES(), esData.getEsParticipants()); - } + esData = elasticSearchable.getParticipantsByRangeAndFilter(ddpInstance.getParticipantIndexES(), participantWrapperPayload.getFrom(), + participantWrapperPayload.getTo(), boolQueryBuilder); + +// if (esData.getEsParticipants().isEmpty()) { +// esData = elasticSearchable.getParticipantsByRangeAndIds(ddpInstance.getParticipantIndexES(), participantWrapperPayload.getFrom(), +// participantWrapperPayload.getTo(), participantIdsToFetch); +// } +// if (participants.isEmpty()) { +// participants = Participant.getParticipantsByIds(ddpInstance.getName(), participantIdsToFetch); +// } +// if (medicalRecords.isEmpty() && ddpInstance.isHasRole()) { +// medicalRecords = MedicalRecord.getMedicalRecordsByParticipantIds(ddpInstance.getName(), participantIdsToFetch); +// } +// if (oncHistoryDetails.isEmpty() && ddpInstance.isHasRole()) { +// oncHistoryDetails = OncHistoryDetail.getOncHistoryDetailsByParticipantIds(ddpInstance.getName(), participantIdsToFetch); +// } +// if (kitRequests.isEmpty() && DDPInstanceDao.getRole(ddpInstance.getName(), DBConstants.KIT_REQUEST_ACTIVATED)) { //only needed if study is shipping samples per DSM +// //get only kitRequests for the filtered pts +// if (Objects.nonNull(esData) && !esData.getEsParticipants().isEmpty()) { +// logger.info("About to query for kits from " + esData.getEsParticipants().size() + " participants"); +// kitRequests = KitRequestShipping.getKitRequestsByParticipantIds(ddpInstance, participantIdsToFetch); +// } +// } + +// if (participantData.isEmpty()) { +// participantData = new ParticipantDataDao().getParticipantDataByParticipantIds(participantIdsToFetch); +// +// // if study is AT +// // TODO - what's the purpose of this if statement +// if ("atcp".equals(ddpInstance.getName())) { +// DefaultValues defaultValues = new DefaultValues(participantData, esData.getEsParticipants(), ddpInstance, null); +// participantData = defaultValues.addDefaultValues(); +// } +// } +// if (abstractionActivities.isEmpty()) { +// abstractionActivities = AbstractionActivity.getAllAbstractionActivityByRealm(ddpInstance.getName()); +// } +// if (abstractionSummary.isEmpty()) { +// abstractionSummary = AbstractionFinal.getAbstractionFinal(ddpInstance.getName()); +// } +// if (proxiesByParticipantIds.isEmpty()) { +// proxiesByParticipantIds = getProxiesWithParticipantIdsFromElasticList(ddpInstance.getUsersIndexES(), esData.getEsParticipants()); +// } + System.out.println(); - System.out.println("awdwa"); + System.out.println(); } private boolean isUnderDsmKey(String source) { diff --git a/src/test/java/org/broadinstitute/dsm/model/participant/ParticipantWrapperTest.java b/src/test/java/org/broadinstitute/dsm/model/participant/ParticipantWrapperTest.java index 1c2f4b649..f264e7adb 100644 --- a/src/test/java/org/broadinstitute/dsm/model/participant/ParticipantWrapperTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/participant/ParticipantWrapperTest.java @@ -15,6 +15,7 @@ import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchable; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; +import org.elasticsearch.index.query.AbstractQueryBuilder; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -121,7 +122,7 @@ public long getParticipantsSize(String esParticipantsIndex) { } @Override - public ElasticSearch getParticipantsByRangeAndFilter(String esParticipantsIndex, int from, int to, String filter) { + public ElasticSearch getParticipantsByRangeAndFilter(String esParticipantsIndex, int from, int to, AbstractQueryBuilder queryBuilder) { return null; } From 5b5f9f58b885bb331355db2d485342d35e1834ce Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 27 Dec 2021 17:18:23 +0400 Subject: [PATCH 263/405] DDP-7121 | introduce DateSplitter, rename old DateSplitter to StrDateSplitter --- .../elastic/export/parse/BaseParser.java | 2 +- .../elastic/filter/AndOrFilterSeparator.java | 2 +- .../model/elastic/filter/DateSplitter.java | 17 ++++----------- .../model/elastic/filter/FilterParser.java | 14 +++++++++++++ .../dsm/model/elastic/filter/Operator.java | 7 +++++-- .../elastic/filter/QueryBuilderFactory.java | 2 +- .../model/elastic/filter/SplitterFactory.java | 3 +++ .../model/elastic/filter/StrDateSplitter.java | 21 +++++++++++++++++++ .../elastic/filter/DateSplitterTest.java | 19 +++++++++++------ .../elastic/filter/FilterParserTest.java | 2 ++ .../elastic/filter/StrDateSplitterTest.java | 19 +++++++++++++++++ 11 files changed, 84 insertions(+), 24 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/StrDateSplitter.java create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/filter/StrDateSplitterTest.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java index 4d6e484d1..252b7f263 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java @@ -83,7 +83,7 @@ public String convertString(String value) { return value; } - private boolean isWrappedByChar(String value) { + protected boolean isWrappedByChar(String value) { return StringUtils.isNotBlank(value) && value.charAt(0) == '\'' && value.charAt(value.length() - 1) == '\''; } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java index 75122d28e..c530308a1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java @@ -11,7 +11,7 @@ public class AndOrFilterSeparator { - public static final String DSM_ALIAS_REGEX = "(m|p|r|t|d|oD|o|k|JS|\\()(\\.|\\s)*([a-z]|O)"; + public static final String DSM_ALIAS_REGEX = "(m|p|r|t|d|oD|o|k|JS|ST|DA|\\()(\\.|\\s)*([a-z]|O|R|T)"; public static final String OR_DSM_ALIAS_REGEX = "(OR) " + DSM_ALIAS_REGEX; public static final String AND_DSM_ALIAS_REGEX = "(AND) " + DSM_ALIAS_REGEX; public static final int AND_PATTERN_MATCHER_NUMBER = 7; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitter.java index db77f7749..07aba52c7 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitter.java @@ -1,21 +1,12 @@ package org.broadinstitute.dsm.model.elastic.filter; -import org.apache.commons.lang3.StringUtils; -import org.broadinstitute.dsm.model.Filter; - public class DateSplitter extends BaseSplitter { + // DATE(FROM_UNIXTIME(k.scan_date/1000)) = DATE(FROM_UNIXTIME(1640563200)) @Override public String[] split() { - // STR_TO_DATE(m.fax_sent,'%Y-%m-%d') = STR_TO_DATE('2021-12-17','%Y-%m-%d') - String[] splittedFilter = new String[2]; - String[] dateFieldWithValue = filter.split(Filter.EQUALS_TRIMMED); - for (int i = 0; i < dateFieldWithValue.length; i++) { - splittedFilter[i] = dateFieldWithValue[i].split(Filter.DATE_FORMAT)[1] - .replace(Filter.OPEN_PARENTHESIS, StringUtils.EMPTY) - .replace(Filter.CLOSE_PARENTHESIS, StringUtils.EMPTY) - .split(",")[0]; - } - return splittedFilter; + + return null; } + } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/FilterParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/FilterParser.java index 366336a56..4df7d1fef 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/FilterParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/FilterParser.java @@ -1,5 +1,6 @@ package org.broadinstitute.dsm.model.elastic.filter; +import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.elastic.export.parse.ValueParser; import java.util.ArrayList; @@ -7,14 +8,23 @@ public class FilterParser extends ValueParser { + // '12313' - str + // 12312 - number + @Override public Object parse(String value) { if (isBoolean(value)) return forBoolean(convertBoolean(value)); + else if (isNumeric(value)) + return forNumber(value); else return convertString(value); } + private boolean isNumeric(String value) { + return !isWrappedByChar(value) && StringUtils.isNumeric(value); + } + @Override public Object[] parse(String[] values) { List parsedValues = new ArrayList<>(); @@ -31,4 +41,8 @@ public Object[] parse(String[] values) { protected Object forNumeric(String value) { return value; } + + private Object forNumber(String value) { + return Long.parseLong(value); + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java index fdd59900f..6e7b85cf5 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java @@ -14,10 +14,11 @@ public enum Operator { IS_NOT_NULL(Filter.IS_NOT_NULL_TRIMMED), DIAMOND_EQUALS(Filter.DIAMOND_EQUALS), MULTIPLE_OPTIONS(Filter.OPEN_PARENTHESIS), - DATE(Filter.DATE_FORMAT), + STR_DATE(Filter.DATE_FORMAT), DATE_GREATER(Filter.DATE_GREATER), DATE_LESS(Filter.DATE_LESS), - JSON_EXTRACT(Filter.JSON_EXTRACT); + JSON_EXTRACT(Filter.JSON_EXTRACT), + DATE(Filter.DATE); private String value; @@ -39,6 +40,8 @@ public static Operator extract(String filter) { else if (filter.startsWith(Filter.OPEN_PARENTHESIS)) return MULTIPLE_OPTIONS; else if (filter.startsWith(Filter.DATE_FORMAT)) + return STR_DATE; + else if (filter.startsWith(Filter.DATE)) return DATE; else if (filter.contains(Filter.DATE_GREATER)) return DATE_GREATER; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java index d14b7ff0b..3189e5011 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java @@ -13,7 +13,7 @@ public static QueryBuilder buildQueryBuilder(Operator operator, QueryPayload pay switch (operator) { case LIKE: case EQUALS: - case DATE: + case STR_DATE: qb = new MatchQueryBuilder(payload.getFieldName(), payload.getValues()[0]); break; case GREATER_THAN_EQUALS: diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java index 6c9e9c141..902f37eb8 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java @@ -26,6 +26,9 @@ public static BaseSplitter createSplitter(Operator operator, String filterValue) case LESS_THAN_EQUALS: splitter = new LessThanEqualsSplitter(); break; + case STR_DATE: + splitter = new StrDateSplitter(); + break; case DATE: splitter = new DateSplitter(); break; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/StrDateSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/StrDateSplitter.java new file mode 100644 index 000000000..53bde3dbf --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/StrDateSplitter.java @@ -0,0 +1,21 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import org.apache.commons.lang3.StringUtils; +import org.broadinstitute.dsm.model.Filter; + +public class StrDateSplitter extends BaseSplitter { + + @Override + public String[] split() { + // STR_TO_DATE(m.fax_sent,'%Y-%m-%d') = STR_TO_DATE('2021-12-17','%Y-%m-%d') + String[] splittedFilter = new String[2]; + String[] dateFieldWithValue = filter.split(Filter.EQUALS_TRIMMED); + for (int i = 0; i < dateFieldWithValue.length; i++) { + splittedFilter[i] = dateFieldWithValue[i].split(Filter.DATE_FORMAT)[1] + .replace(Filter.OPEN_PARENTHESIS, StringUtils.EMPTY) + .replace(Filter.CLOSE_PARENTHESIS, StringUtils.EMPTY) + .split(",")[0]; + } + return splittedFilter; + } +} diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitterTest.java index a93f76d16..25af55b70 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitterTest.java @@ -3,17 +3,24 @@ import org.junit.Assert; import org.junit.Test; +import static org.junit.Assert.*; + public class DateSplitterTest { @Test public void split() { - String filter = "STR_TO_DATE(m.fax_sent,'%Y-%m-%d') = STR_TO_DATE('2021-12-17','%Y-%m-%d')"; - BaseSplitter dateSplitter = SplitterFactory.createSplitter(Operator.DATE, filter); - dateSplitter.setFilter(filter); + String filter = "DATE(FROM_UNIXTIME(k.scan_date/1000)) = DATE(FROM_UNIXTIME(1640563200))"; + BaseSplitter splitter = SplitterFactory.createSplitter(Operator.DATE, filter); + splitter.setFilter(filter); + +// Assert.assertEquals("1640563200"); + splitter.getValue(); +// splitter.(); + splitter.getValue(); + +// '123123' + - Assert.assertEquals("m", dateSplitter.getAlias()); - Assert.assertEquals("faxSent", dateSplitter.getInnerProperty()); - Assert.assertEquals("'2021-12-17'", dateSplitter.getValue()[0]); } } \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/FilterParserTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/FilterParserTest.java index c1be91a1d..f74bb078a 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/FilterParserTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/FilterParserTest.java @@ -14,12 +14,14 @@ public void parse() { String str = "'string'"; String number = "'5'"; String date = "'1999-05-22'"; + String number2 = "5"; BaseParser filterParser = new FilterParser(); Assert.assertEquals(true, filterParser.parse(trueValue)); Assert.assertEquals(false, filterParser.parse(falseValue)); Assert.assertEquals("string", filterParser.parse(str)); Assert.assertEquals("5", filterParser.parse(number)); + Assert.assertEquals(5, filterParser.parse(number2)); Assert.assertEquals("1999-05-22", filterParser.parse(date)); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/StrDateSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/StrDateSplitterTest.java new file mode 100644 index 000000000..8b6262180 --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/StrDateSplitterTest.java @@ -0,0 +1,19 @@ +package org.broadinstitute.dsm.model.elastic.filter; + +import org.junit.Assert; +import org.junit.Test; + +public class StrDateSplitterTest { + + @Test + public void split() { + + String filter = "STR_TO_DATE(m.fax_sent,'%Y-%m-%d') = STR_TO_DATE('2021-12-17','%Y-%m-%d')"; + BaseSplitter dateSplitter = SplitterFactory.createSplitter(Operator.STR_DATE, filter); + dateSplitter.setFilter(filter); + + Assert.assertEquals("m", dateSplitter.getAlias()); + Assert.assertEquals("faxSent", dateSplitter.getInnerProperty()); + Assert.assertEquals("'2021-12-17'", dateSplitter.getValue()[0]); + } +} \ No newline at end of file From 0d98042af65fa746c99cb913b7bd03cc32cd320e Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 27 Dec 2021 17:50:09 +0400 Subject: [PATCH 264/405] DDP-7121 | add Date splitter --- .../dsm/model/elastic/filter/DateSplitter.java | 12 +++++++++--- .../dsm/model/elastic/filter/FilterParser.java | 2 ++ .../dsm/model/elastic/filter/DateSplitterTest.java | 9 ++------- .../dsm/model/elastic/filter/FilterParserTest.java | 2 +- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitter.java index 07aba52c7..9a32b3187 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitter.java @@ -1,12 +1,18 @@ package org.broadinstitute.dsm.model.elastic.filter; -public class DateSplitter extends BaseSplitter { +import org.broadinstitute.dsm.model.Filter; + +public class DateSplitter extends EqualsSplitter { // DATE(FROM_UNIXTIME(k.scan_date/1000)) = DATE(FROM_UNIXTIME(1640563200)) @Override public String[] split() { - - return null; + String[] equalSeparated = super.split(); + String leftSide = equalSeparated[0].trim(); + String rightSide = equalSeparated[1].trim(); + String fieldWithAlias = leftSide.split("\\" + Filter.OPEN_PARENTHESIS)[2].split("/")[0]; + String value = rightSide.split("\\" + Filter.OPEN_PARENTHESIS)[2].split("\\" + Filter.CLOSE_PARENTHESIS)[0]; + return new String[] {fieldWithAlias, value}; } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/FilterParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/FilterParser.java index 4df7d1fef..1a45895e1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/FilterParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/FilterParser.java @@ -31,6 +31,8 @@ public Object[] parse(String[] values) { for (String value : values) { if (isBoolean(value)) parsedValues.add(forBoolean(convertBoolean(value))); + else if (isNumeric(value)) + parsedValues.add(forNumber(value)); else parsedValues.add(convertString(value)); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitterTest.java index 25af55b70..79604aa0f 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitterTest.java @@ -14,13 +14,8 @@ public void split() { BaseSplitter splitter = SplitterFactory.createSplitter(Operator.DATE, filter); splitter.setFilter(filter); -// Assert.assertEquals("1640563200"); - splitter.getValue(); -// splitter.(); - splitter.getValue(); - -// '123123' - + Assert.assertEquals("1640563200", splitter.getValue()[0]); + Assert.assertEquals("scanDate", splitter.getInnerProperty()); } } \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/FilterParserTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/FilterParserTest.java index f74bb078a..a92ef3ab3 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/FilterParserTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/FilterParserTest.java @@ -21,7 +21,7 @@ public void parse() { Assert.assertEquals(false, filterParser.parse(falseValue)); Assert.assertEquals("string", filterParser.parse(str)); Assert.assertEquals("5", filterParser.parse(number)); - Assert.assertEquals(5, filterParser.parse(number2)); + Assert.assertEquals(5L, filterParser.parse(number2)); Assert.assertEquals("1999-05-22", filterParser.parse(date)); } From d3f2c8f1ce34612cf458077a054e438b8396d74a Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 27 Dec 2021 18:34:09 +0400 Subject: [PATCH 265/405] DDP-7121 | add DATE case --- src/main/java/org/broadinstitute/dsm/model/Filter.java | 1 + .../dsm/model/elastic/filter/QueryBuilderFactory.java | 1 + 2 files changed, 2 insertions(+) diff --git a/src/main/java/org/broadinstitute/dsm/model/Filter.java b/src/main/java/org/broadinstitute/dsm/model/Filter.java index f6d5854ad..e68d7e370 100644 --- a/src/main/java/org/broadinstitute/dsm/model/Filter.java +++ b/src/main/java/org/broadinstitute/dsm/model/Filter.java @@ -304,6 +304,7 @@ private static String generateDateComparisonSql(Filter filter, DBElement dbEleme dateConverter = dbElement.getDateConverter(); Instant instant = null; try { + // 29/07/2021 LocalDate date = LocalDate.parse(arg.toString(), DateTimeFormatter.ISO_LOCAL_DATE); instant = useEndOfday ? date.atTime(LocalTime.MAX).toInstant(ZoneOffset.UTC) : date.atStartOfDay().toInstant(ZoneOffset.UTC); } catch (DateTimeParseException e) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java index 3189e5011..af94a77bb 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java @@ -13,6 +13,7 @@ public static QueryBuilder buildQueryBuilder(Operator operator, QueryPayload pay switch (operator) { case LIKE: case EQUALS: + case DATE: case STR_DATE: qb = new MatchQueryBuilder(payload.getFieldName(), payload.getValues()[0]); break; From fdef1516e289320f329665d37614033cd05cc858 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 28 Dec 2021 10:54:04 +0400 Subject: [PATCH 266/405] DDP-7121 | start mapping data from es to dtos --- .../model/participant/ParticipantWrapper.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java index 2f1e168ab..e22f2df2d 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java @@ -292,6 +292,27 @@ private List collectData() { logger.info("Collecting participant data..."); List result = new ArrayList<>(); for (ElasticSearchParticipantDto elasticSearchParticipantDto: esData.getEsParticipants()) { + + elasticSearchParticipantDto.getDsm().ifPresent(esDsm -> { + ParticipantWrapperDto participantWrapperDto = new ParticipantWrapperDto(); + participantWrapperDto.setEsData(elasticSearchParticipantDto); + + Participant participant = esDsm.getParticipant(); + participant.setCreatedOncHistory(esDsm.getOncHistory().getCreated()); + participant.setReviewedOncHistory(esDsm.getOncHistory().getReviewed()); + + List oncHistoryDetails = esDsm.getOncHistoryDetail(); + List tissues = esDsm.getTissue(); + for (Tissue tissue : tissues) { + String oncHistoryDetailId = tissue.getOncHistoryDetailId(); + oncHistoryDetails.stream() + .filter(oncHistoryDetail -> oncHistoryDetail.getOncHistoryDetailId().equals(oncHistoryDetailId)) + .findFirst() + .ifPresent(oncHistoryDetail -> oncHistoryDetail.getTissues().add(tissue)); + } + + List kitRequestShipping = esDsm.getKitRequestShipping(); + }); String participantId = elasticSearchParticipantDto.getParticipantId(); if (StringUtils.isBlank(participantId)) continue; Participant participant = participants.stream() From d307ae78cb987ad2c47ac274d4dafaaba7ce84ac Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 28 Dec 2021 12:39:20 +0400 Subject: [PATCH 267/405] DDP-7121 | write getters for DTO classes --- .../dsm/db/OncHistoryDetail.java | 7 +++ .../broadinstitute/dsm/db/Participant.java | 7 +-- .../dsm/model/elastic/ESDsm.java | 9 +++ .../elastic/filter/AndOrFilterSeparator.java | 6 +- .../model/participant/ParticipantWrapper.java | 63 +++++++++++++------ .../filter/AndOrFilterSeparatorTest.java | 10 +++ 6 files changed, 76 insertions(+), 26 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java b/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java index e37414b33..02039fa09 100644 --- a/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java +++ b/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java @@ -313,6 +313,13 @@ public void addTissue(Tissue tissue) { } } + public List getTissues() { + if (tissues == null) { + tissues = new ArrayList<>(); + } + return tissues; + } + public static Map> getOncHistoryDetails(@NonNull String realm) { return getOncHistoryDetails(realm, null); } diff --git a/src/main/java/org/broadinstitute/dsm/db/Participant.java b/src/main/java/org/broadinstitute/dsm/db/Participant.java index b4b5e7eee..9d3b1f9d6 100644 --- a/src/main/java/org/broadinstitute/dsm/db/Participant.java +++ b/src/main/java/org/broadinstitute/dsm/db/Participant.java @@ -4,7 +4,6 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.annotations.SerializedName; import lombok.Data; import lombok.NonNull; @@ -104,7 +103,7 @@ public class Participant { primaryKey = DBConstants.PARTICIPANT_ID, columnPrefix = "") @ColumnName (DBConstants.NOTES) - private String ptNotes; + private String notes; @TableName ( name = DBConstants.DDP_PARTICIPANT_RECORD, @@ -152,7 +151,7 @@ public Map getDynamicFields() { public Participant() {} public Participant(String participantId, String ddpParticipantId, String assigneeIdMr, String assigneeIdTissue, String instanceName, - String createdOncHistory, String reviewedOncHistory, String paperCRSent, String paperCRReceived, String ptNotes, + String createdOncHistory, String reviewedOncHistory, String paperCRSent, String paperCRReceived, String notes, boolean minimalMr, boolean abstractionReady, String additionalValuesJson, long exitDate) { this.participantId = participantId; this.ddpParticipantId = ddpParticipantId; @@ -163,7 +162,7 @@ public Participant(String participantId, String ddpParticipantId, String assigne this.reviewedOncHistory = reviewedOncHistory; this.paperCRSent = paperCRSent; this.paperCRReceived = paperCRReceived; - this.ptNotes = ptNotes; + this.notes = notes; this.minimalMr = minimalMr; this.abstractionReady = abstractionReady; this.additionalValuesJson = additionalValuesJson; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java index d6988a0ab..f010387f3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java @@ -2,6 +2,7 @@ import java.util.Collections; import java.util.List; +import java.util.Optional; import com.google.gson.annotations.SerializedName; import lombok.Getter; @@ -81,4 +82,12 @@ public List getKitRequestShipping() { if (kitRequestShipping == null) kitRequestShipping = Collections.emptyList(); return kitRequestShipping; } + + public Optional getOncHistory() { + return Optional.ofNullable(oncHistory); + } + + public Optional getParticipant() { + return Optional.ofNullable(participant); + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java index c530308a1..7c5db87cf 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java @@ -11,11 +11,11 @@ public class AndOrFilterSeparator { - public static final String DSM_ALIAS_REGEX = "(m|p|r|t|d|oD|o|k|JS|ST|DA|\\()(\\.|\\s)*([a-z]|O|R|T)"; + public static final String DSM_ALIAS_REGEX = "(m|p|r|t|d|oD|o|k|JS|ST|DA|\\()(\\.|\\s)*([a-z]|O|R|T){1,2}"; public static final String OR_DSM_ALIAS_REGEX = "(OR) " + DSM_ALIAS_REGEX; public static final String AND_DSM_ALIAS_REGEX = "(AND) " + DSM_ALIAS_REGEX; - public static final int AND_PATTERN_MATCHER_NUMBER = 7; - public static final int OR_PATTERN_MATCHER_NUMBER = 6; + public static final int AND_PATTERN_MATCHER_NUMBER = 8; + public static final int OR_PATTERN_MATCHER_NUMBER = 7; public static final int MINIMUM_STEP_FROM_OPERATOR = 3; private String filter; diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java index e22f2df2d..6f887b4f8 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java @@ -70,7 +70,7 @@ public ParticipantWrapperResult getFilteredList() { .map(filters -> { fetchAndPrepareDataByFilters(ddpInstance, filters); sortBySelfElseById(participantData.values()); - return new ParticipantWrapperResult(esData.getTotalCount(), collectData()); + return new ParticipantWrapperResult(esData.getTotalCount(), collectData(ddpInstance)); }) .orElseGet(() -> { fetchAndPrepareData(ddpInstance); @@ -80,7 +80,7 @@ public ParticipantWrapperResult getFilteredList() { participantData = defaultValues.addDefaultValues(); } sortBySelfElseById(participantData.values()); - return new ParticipantWrapperResult(esData.getTotalCount(), collectData()); + return new ParticipantWrapperResult(esData.getTotalCount(), collectData(ddpInstance)); }); } @@ -276,7 +276,7 @@ private List getParticipantsFromEsData() { List participants = new ArrayList<>(); for (ElasticSearchParticipantDto elasticSearchParticipantDto: esData.getEsParticipants()) { elasticSearchParticipantDto.getDsm().ifPresent(esDsm -> { - Participant participant = esDsm.getParticipant(); + Participant participant = esDsm.getParticipant().orElse(new Participant()); if (Objects.nonNull(participant)) participants.add(participant); }); } @@ -288,18 +288,21 @@ Map> getProxiesWithParticipantIdsFromE return getProxiesWithParticipantIdsByProxiesIds(esUsersIndex, proxiesIdsFromElasticList); } - private List collectData() { + private List collectData(DDPInstance ddpInstance) { logger.info("Collecting participant data..."); List result = new ArrayList<>(); for (ElasticSearchParticipantDto elasticSearchParticipantDto: esData.getEsParticipants()) { elasticSearchParticipantDto.getDsm().ifPresent(esDsm -> { - ParticipantWrapperDto participantWrapperDto = new ParticipantWrapperDto(); - participantWrapperDto.setEsData(elasticSearchParticipantDto); - Participant participant = esDsm.getParticipant(); - participant.setCreatedOncHistory(esDsm.getOncHistory().getCreated()); - participant.setReviewedOncHistory(esDsm.getOncHistory().getReviewed()); + Participant participant = esDsm.getParticipant().orElse(new Participant()); + + esDsm.getOncHistory().ifPresent(oncHistory -> { + participant.setCreatedOncHistory(oncHistory.getCreated()); + participant.setReviewedOncHistory(oncHistory.getReviewed()); + }); + + List medicalRecord = esDsm.getMedicalRecord(); List oncHistoryDetails = esDsm.getOncHistoryDetail(); List tissues = esDsm.getTissue(); @@ -312,17 +315,39 @@ private List collectData() { } List kitRequestShipping = esDsm.getKitRequestShipping(); + + List proxyGuids = elasticSearchParticipantDto.getProxies(); + String usersIndexES = ddpInstance.getUsersIndexES(); + ElasticSearch participantsByIds = elasticSearchable.getParticipantsByIds(usersIndexES, proxyGuids); + List proxies = participantsByIds.getEsParticipants(); + + List participantData = esDsm.getParticipantData(); + + + ParticipantWrapperDto participantWrapperDto = new ParticipantWrapperDto(); + participantWrapperDto.setEsData(elasticSearchParticipantDto); + participantWrapperDto.setParticipant(participant); + participantWrapperDto.setMedicalRecords(medicalRecord); + participantWrapperDto.setOncHistoryDetails(oncHistoryDetails); + participantWrapperDto.setKits(kitRequestShipping); + participantWrapperDto.setProxyData(proxies); + participantWrapperDto.setParticipantData(participantData); + participantWrapperDto.setAbstractionActivities(Collections.emptyList()); + participantWrapperDto.setAbstractionSummary(Collections.emptyList()); + + result.add(participantWrapperDto); + }); - String participantId = elasticSearchParticipantDto.getParticipantId(); - if (StringUtils.isBlank(participantId)) continue; - Participant participant = participants.stream() - .filter(ppt -> participantId.equals(ppt.getDdpParticipantId())) - .findFirst() - .orElse(null); - result.add(new ParticipantWrapperDto( - elasticSearchParticipantDto, participant, medicalRecords.get(participantId), - oncHistoryDetails.get(participantId), kitRequests.get(participantId), abstractionActivities.get(participantId), - abstractionSummary.get(participantId), proxiesByParticipantIds.get(participantId), participantData.get(participantId))); +// String participantId = elasticSearchParticipantDto.getParticipantId(); +// if (StringUtils.isBlank(participantId)) continue; +// Participant participant = participants.stream() +// .filter(ppt -> participantId.equals(ppt.getDdpParticipantId())) +// .findFirst() +// .orElse(null); +// result.add(new ParticipantWrapperDto( +// elasticSearchParticipantDto, participant, medicalRecords.get(participantId), +// oncHistoryDetails.get(participantId), kitRequests.get(participantId), abstractionActivities.get(participantId), +// abstractionSummary.get(participantId), proxiesByParticipantIds.get(participantId), participantData.get(participantId))); } return result; } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparatorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparatorTest.java index d1713d8fe..f30ba3c46 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparatorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparatorTest.java @@ -42,4 +42,14 @@ public void parseFiltersByLogicalOperators() { } } } + + @Test + public void parseFiltersByLogicalOperatorsSingle() { + + String filter = "AND oD.datePx = '15'"; + Map> stringListMap = new AndOrFilterSeparator(filter).parseFiltersByLogicalOperators(); + Assert.assertEquals("oD.datePx = '15'", stringListMap.get("AND").get(0)); + + + } } \ No newline at end of file From 321c33e192dd5c029ede8606856930fc67a9454d Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 28 Dec 2021 16:31:05 +0400 Subject: [PATCH 268/405] DDP-7121 | attempt to fix optional filters problem --- .../elastic/filter/AndOrFilterSeparator.java | 23 +++++++++++++++---- .../filter/AndOrFilterSeparatorTest.java | 9 +++++--- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java index 7c5db87cf..373c80282 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java @@ -11,11 +11,11 @@ public class AndOrFilterSeparator { - public static final String DSM_ALIAS_REGEX = "(m|p|r|t|d|oD|o|k|JS|ST|DA|\\()(\\.|\\s)*([a-z]|O|R|T){1,2}"; + public static final String DSM_ALIAS_REGEX = "(m|p|r|t|d|oD|o|k|JS|ST|DA|\\()(\\.|\\s)*([a-z]|O|R|T)"; public static final String OR_DSM_ALIAS_REGEX = "(OR) " + DSM_ALIAS_REGEX; public static final String AND_DSM_ALIAS_REGEX = "(AND) " + DSM_ALIAS_REGEX; - public static final int AND_PATTERN_MATCHER_NUMBER = 8; - public static final int OR_PATTERN_MATCHER_NUMBER = 7; + public static final int AND_PATTERN_MATCHER_NUMBER = 7; + public static final int OR_PATTERN_MATCHER_NUMBER = 6; public static final int MINIMUM_STEP_FROM_OPERATOR = 3; private String filter; @@ -67,12 +67,27 @@ private void handleSpecialCases(Map> filterByLogicalOperato */ private int findProperOperatorSplitterIndex(String operator, int startIndex, int patternMatcherNumber, int nextOperatorFromNumber) { String aliasRegex = getAliasRegexByOperator(operator); - while (startIndex != -1 && !filter.substring(startIndex, startIndex + patternMatcherNumber).matches(aliasRegex)) { + while (startIndex != -1 + && !filter.substring(startIndex, startIndex + patternMatcherNumber).matches(aliasRegex) + && !isLeftSideOpeningParenthesisPresent(startIndex)){ startIndex = findNextOperatorIndex(operator, startIndex + nextOperatorFromNumber); } return startIndex; } + private boolean isLeftSideOpeningParenthesisPresent(int startIndex) { + boolean exists = false; + for (int i = startIndex; i > 0; i++) { + char c = filter.charAt(i); + if (c == '(') { + exists = true; + break; + } + if (Filter.AND_TRIMMED.equals(filter.substring(i-3, i)) || c == ')') break; + } + return exists; + } + private String getAliasRegexByOperator(String operator) { return "AND".equals(operator) ? AND_DSM_ALIAS_REGEX : OR_DSM_ALIAS_REGEX; } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparatorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparatorTest.java index f30ba3c46..feda4d687 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparatorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparatorTest.java @@ -20,10 +20,11 @@ public void parseFiltersByLogicalOperators() { "OR m.medicalRecordSomethingg = '55552' " + "AND t.tissueRecord IS NOT NULL " + "AND m.dynamicFields.ragac = '55' " + + "AND ( t.tissue = 'review' OR t.tissue = 'no' OR t.tissue = 'bla' ) " + "OR m.medicalRecordName = '213' " + "OR m.mrNotes = 'MEDICAL_RECORD_NOTESS' " + "AND m.medicalMedical = 'something AND something' " + - "AND ( oD.request = 'review' OR oD.request = 'no' ) " + + "AND ( oD.request = 'review' OR oD.request = 'no' OR oD.request = 'bla' ) " + "OR t.tissueRecord = '225' " + "AND JSON_EXTRACT ( m.additiona`l_values_json , '$.seeingIfBugExists' )"; AndOrFilterSeparator andOrFilterSeparator = new AndOrFilterSeparator(filter); @@ -31,8 +32,10 @@ public void parseFiltersByLogicalOperators() { for (Map.Entry> eachFilter: parsedFilters.entrySet()) { if (eachFilter.getKey().equals("AND")) { Assert.assertArrayEquals(new ArrayList<>(List.of("m.medicalRecordId = '15'", - "t.tissueRecord IS NOT NULL" ,"m.dynamicFields.ragac = '55'", "m" + - ".medicalMedical = 'something AND something'", "( oD.request = 'review' OR oD.request = 'no' )", + "t.tissueRecord IS NOT NULL" ,"m.dynamicFields.ragac = '55'", + "( t.tissue = 'review' OR t.tissue = 'no' OR t.tissue = 'bla' )", + "m.medicalMedical = 'something AND something'", "( oD.request = 'review' OR oD.request = 'no' OR " + + "oD.request = 'bla' )", "JSON_EXTRACT ( m.additiona`l_values_json , '$.seeingIfBugExists' )")).toArray(), eachFilter.getValue().toArray()); } else { From 2c6c0ecdc9eace0a049afb8d8b21753ac0a30138 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 29 Dec 2021 11:06:09 +0400 Subject: [PATCH 269/405] DDP-7121 | refactor to match oD case --- .../elastic/filter/AndOrFilterSeparator.java | 60 +++++++++++++++---- .../filter/AndOrFilterSeparatorTest.java | 17 +++++- 2 files changed, 61 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java index 373c80282..559940dbc 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java @@ -11,7 +11,7 @@ public class AndOrFilterSeparator { - public static final String DSM_ALIAS_REGEX = "(m|p|r|t|d|oD|o|k|JS|ST|DA|\\()(\\.|\\s)*([a-z]|O|R|T)"; + public static final String DSM_ALIAS_REGEX = "(m|p|r|t|d|o|k|JS|ST|DA|\\()(\\.|\\s)*([a-z]|O|R|T|D|)(\\.)*"; public static final String OR_DSM_ALIAS_REGEX = "(OR) " + DSM_ALIAS_REGEX; public static final String AND_DSM_ALIAS_REGEX = "(AND) " + DSM_ALIAS_REGEX; public static final int AND_PATTERN_MATCHER_NUMBER = 7; @@ -68,8 +68,8 @@ private void handleSpecialCases(Map> filterByLogicalOperato private int findProperOperatorSplitterIndex(String operator, int startIndex, int patternMatcherNumber, int nextOperatorFromNumber) { String aliasRegex = getAliasRegexByOperator(operator); while (startIndex != -1 - && !filter.substring(startIndex, startIndex + patternMatcherNumber).matches(aliasRegex) - && !isLeftSideOpeningParenthesisPresent(startIndex)){ + && (isLeftSideOpeningParenthesisPresent(startIndex) + || !filter.substring(startIndex, startIndex + patternMatcherNumber).matches(aliasRegex))){ startIndex = findNextOperatorIndex(operator, startIndex + nextOperatorFromNumber); } return startIndex; @@ -77,7 +77,7 @@ private int findProperOperatorSplitterIndex(String operator, int startIndex, int private boolean isLeftSideOpeningParenthesisPresent(int startIndex) { boolean exists = false; - for (int i = startIndex; i > 0; i++) { + for (int i = startIndex; i > 2; i--) { char c = filter.charAt(i); if (c == '(') { exists = true; @@ -93,7 +93,12 @@ private String getAliasRegexByOperator(String operator) { } private int findNextOperatorIndex(String operator, int fromIndex) { - return filter.indexOf(operator, fromIndex); + int index = filter.indexOf(operator, fromIndex); + if (isLeftSideOpeningParenthesisPresent(index)) { + index = filter.indexOf(Filter.CLOSE_PARENTHESIS, index); + index = filter.indexOf(operator, index); + } + return index; } private int getIndex(Map> filterByLogicalOperators, int index, String operator) { @@ -105,16 +110,45 @@ private int getIndex(Map> filterByLogicalOperators, int ind int andPrecedeIndex = findNextOperatorIndex(Filter.AND_TRIMMED, index + MINIMUM_STEP_FROM_OPERATOR); andPrecedeIndex = findProperOperatorSplitterIndex(Filter.AND_TRIMMED, andPrecedeIndex, AND_PATTERN_MATCHER_NUMBER, MINIMUM_STEP_FROM_OPERATOR); - if (isLeftMostOR(matches, orPrecedeIndex, andPrecedeIndex)) { - filterByLogicalOperators.get(operator).add(filter.substring(index + MINIMUM_STEP_FROM_OPERATOR, orPrecedeIndex == -1 ? andPrecedeIndex : orPrecedeIndex).trim()); - index = isAndOperator(operator) ? andPrecedeIndex : orPrecedeIndex; - } else if (isLeftMostAND(matches, orPrecedeIndex, andPrecedeIndex)) { - filterByLogicalOperators.get(operator).add(filter.substring(index + MINIMUM_STEP_FROM_OPERATOR, andPrecedeIndex == -1 ? orPrecedeIndex : andPrecedeIndex).trim()); - index = isAndOperator(operator) ? andPrecedeIndex : orPrecedeIndex; + if (andPrecedeIndex > index && orPrecedeIndex > index) { + filterByLogicalOperators.get(operator).add(filter.substring(index + MINIMUM_STEP_FROM_OPERATOR, Math.min(orPrecedeIndex, + andPrecedeIndex)).trim()); + index = Math.min(orPrecedeIndex, + andPrecedeIndex); } else { - filterByLogicalOperators.get(operator).add(filter.substring(index + MINIMUM_STEP_FROM_OPERATOR).trim()); - index = isAndOperator(operator) ? andPrecedeIndex : orPrecedeIndex; + if (andPrecedeIndex == -1 && orPrecedeIndex == -1) { + filterByLogicalOperators.get(operator).add(filter.substring(index + MINIMUM_STEP_FROM_OPERATOR).trim()); + return -1; + } + filterByLogicalOperators.get(operator).add(filter.substring(index + MINIMUM_STEP_FROM_OPERATOR, Math.max(orPrecedeIndex, + andPrecedeIndex)).trim()); + index = Math.max(orPrecedeIndex, + andPrecedeIndex); } + +// if (isLeftMostOR(matches, orPrecedeIndex, andPrecedeIndex)) { +// if (orPrecedeIndex > index && andPrecedeIndex > index) { +// filterByLogicalOperators.get(operator).add(filter.substring(index + MINIMUM_STEP_FROM_OPERATOR, Math.min(orPrecedeIndex, +// andPrecedeIndex)).trim()); +// } else { +// filterByLogicalOperators.get(operator).add(filter.substring(index + MINIMUM_STEP_FROM_OPERATOR, Math.max(orPrecedeIndex, +// andPrecedeIndex)).trim()); +// } +// index = isAndOperator(operator) ? andPrecedeIndex : orPrecedeIndex; +// } else if (isLeftMostAND(matches, orPrecedeIndex, andPrecedeIndex)) { +// if (orPrecedeIndex > index && andPrecedeIndex > index) { +// filterByLogicalOperators.get(operator).add(filter.substring(index + MINIMUM_STEP_FROM_OPERATOR, Math.min(orPrecedeIndex, +// andPrecedeIndex)).trim()); +// } else { +// filterByLogicalOperators.get(operator).add(filter.substring(index + MINIMUM_STEP_FROM_OPERATOR, Math.max(orPrecedeIndex, +// andPrecedeIndex)).trim()); +// } +//// filterByLogicalOperators.get(operator).add(filter.substring(index + MINIMUM_STEP_FROM_OPERATOR, andPrecedeIndex == -1 ? orPrecedeIndex : andPrecedeIndex).trim()); +// index = isAndOperator(operator) ? andPrecedeIndex : orPrecedeIndex; +// } else { +// filterByLogicalOperators.get(operator).add(filter.substring(index + MINIMUM_STEP_FROM_OPERATOR).trim()); +// index = isAndOperator(operator) ? andPrecedeIndex : orPrecedeIndex; +// } return index; } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparatorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparatorTest.java index feda4d687..b06e22fb0 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparatorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparatorTest.java @@ -49,10 +49,21 @@ public void parseFiltersByLogicalOperators() { @Test public void parseFiltersByLogicalOperatorsSingle() { - String filter = "AND oD.datePx = '15'"; + String filter = "AND oD.datePx = '15' "; Map> stringListMap = new AndOrFilterSeparator(filter).parseFiltersByLogicalOperators(); Assert.assertEquals("oD.datePx = '15'", stringListMap.get("AND").get(0)); - } -} \ No newline at end of file + +} + + + + + + + + + + + From c0bc29997e0328f9847205a8cb3dce1e31e5cbf7 Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 29 Dec 2021 11:37:33 +0400 Subject: [PATCH 270/405] DDP-7121 | start refactoring AndOrFilterSeparator.java --- .../elastic/filter/AndOrFilterSeparator.java | 78 ++++++++----------- 1 file changed, 33 insertions(+), 45 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java index 559940dbc..67aabcc0f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java @@ -6,7 +6,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.regex.Pattern; import java.util.stream.Collectors; public class AndOrFilterSeparator { @@ -29,8 +28,7 @@ public void setFilter(String filter) { } protected Map> parseFiltersByLogicalOperators() { - Map> filterByLogicalOperators = new ConcurrentHashMap<>(Map.of(Filter.AND_TRIMMED, new ArrayList<>(), Filter.OR_TRIMMED, - new ArrayList<>())); + Map> filterByLogicalOperators = new ConcurrentHashMap<>(Map.of(Filter.AND_TRIMMED, new ArrayList<>(), Filter.OR_TRIMMED, new ArrayList<>())); int andIndex = filter.indexOf(Filter.AND_TRIMMED); int orIndex = filter.indexOf(Filter.OR_TRIMMED); while (andIndex != -1 || orIndex != -1) { @@ -68,22 +66,25 @@ private void handleSpecialCases(Map> filterByLogicalOperato private int findProperOperatorSplitterIndex(String operator, int startIndex, int patternMatcherNumber, int nextOperatorFromNumber) { String aliasRegex = getAliasRegexByOperator(operator); while (startIndex != -1 - && (isLeftSideOpeningParenthesisPresent(startIndex) - || !filter.substring(startIndex, startIndex + patternMatcherNumber).matches(aliasRegex))){ + && (isOperatorWrappedInParenthesis(startIndex) + || !isMatches(startIndex, startIndex + patternMatcherNumber, aliasRegex))){ startIndex = findNextOperatorIndex(operator, startIndex + nextOperatorFromNumber); } return startIndex; } - private boolean isLeftSideOpeningParenthesisPresent(int startIndex) { + private boolean isOperatorWrappedInParenthesis(int startIndex) { + final char openParenthesis = '('; + final char closeParenthesis = ')'; boolean exists = false; for (int i = startIndex; i > 2; i--) { char c = filter.charAt(i); - if (c == '(') { + if (c == openParenthesis) { exists = true; break; } - if (Filter.AND_TRIMMED.equals(filter.substring(i-3, i)) || c == ')') break; + if (Filter.AND_TRIMMED.equals(filter.substring(i - 3, i)) || c == closeParenthesis) break; + // ) OR } return exists; } @@ -94,7 +95,7 @@ private String getAliasRegexByOperator(String operator) { private int findNextOperatorIndex(String operator, int fromIndex) { int index = filter.indexOf(operator, fromIndex); - if (isLeftSideOpeningParenthesisPresent(index)) { + if (isOperatorWrappedInParenthesis(index)) { index = filter.indexOf(Filter.CLOSE_PARENTHESIS, index); index = filter.indexOf(operator, index); } @@ -102,7 +103,6 @@ private int findNextOperatorIndex(String operator, int fromIndex) { } private int getIndex(Map> filterByLogicalOperators, int index, String operator) { - boolean matches = filter.substring(index, getFilterIndex(index, getPatternMatcherNumberByOperator(operator))).matches(Pattern.compile(getAliasRegexByOperator(operator)).pattern()); int orPrecedeIndex = findNextOperatorIndex(Filter.OR_TRIMMED, index + MINIMUM_STEP_FROM_OPERATOR); orPrecedeIndex = findProperOperatorSplitterIndex(Filter.OR_TRIMMED, orPrecedeIndex, OR_PATTERN_MATCHER_NUMBER, MINIMUM_STEP_FROM_OPERATOR); @@ -110,48 +110,36 @@ private int getIndex(Map> filterByLogicalOperators, int ind int andPrecedeIndex = findNextOperatorIndex(Filter.AND_TRIMMED, index + MINIMUM_STEP_FROM_OPERATOR); andPrecedeIndex = findProperOperatorSplitterIndex(Filter.AND_TRIMMED, andPrecedeIndex, AND_PATTERN_MATCHER_NUMBER, MINIMUM_STEP_FROM_OPERATOR); - if (andPrecedeIndex > index && orPrecedeIndex > index) { - filterByLogicalOperators.get(operator).add(filter.substring(index + MINIMUM_STEP_FROM_OPERATOR, Math.min(orPrecedeIndex, - andPrecedeIndex)).trim()); - index = Math.min(orPrecedeIndex, - andPrecedeIndex); + // index = 5, 4 ,20 + if (isAndOrGreaterThanCurrentPosition(index, orPrecedeIndex, andPrecedeIndex)) { + filterByLogicalOperators.get(operator).add(filter.substring(index + MINIMUM_STEP_FROM_OPERATOR, Math.min(orPrecedeIndex, andPrecedeIndex)).trim()); + index = Math.min(orPrecedeIndex, andPrecedeIndex); } else { - if (andPrecedeIndex == -1 && orPrecedeIndex == -1) { + if (isEndOfFilter(orPrecedeIndex, andPrecedeIndex)) { filterByLogicalOperators.get(operator).add(filter.substring(index + MINIMUM_STEP_FROM_OPERATOR).trim()); - return -1; + index = -1; + } else { + filterByLogicalOperators.get(operator).add(filter.substring(index + MINIMUM_STEP_FROM_OPERATOR, Math.max(orPrecedeIndex, + andPrecedeIndex)).trim()); + index = Math.max(orPrecedeIndex, + andPrecedeIndex); } - filterByLogicalOperators.get(operator).add(filter.substring(index + MINIMUM_STEP_FROM_OPERATOR, Math.max(orPrecedeIndex, - andPrecedeIndex)).trim()); - index = Math.max(orPrecedeIndex, - andPrecedeIndex); } - -// if (isLeftMostOR(matches, orPrecedeIndex, andPrecedeIndex)) { -// if (orPrecedeIndex > index && andPrecedeIndex > index) { -// filterByLogicalOperators.get(operator).add(filter.substring(index + MINIMUM_STEP_FROM_OPERATOR, Math.min(orPrecedeIndex, -// andPrecedeIndex)).trim()); -// } else { -// filterByLogicalOperators.get(operator).add(filter.substring(index + MINIMUM_STEP_FROM_OPERATOR, Math.max(orPrecedeIndex, -// andPrecedeIndex)).trim()); -// } -// index = isAndOperator(operator) ? andPrecedeIndex : orPrecedeIndex; -// } else if (isLeftMostAND(matches, orPrecedeIndex, andPrecedeIndex)) { -// if (orPrecedeIndex > index && andPrecedeIndex > index) { -// filterByLogicalOperators.get(operator).add(filter.substring(index + MINIMUM_STEP_FROM_OPERATOR, Math.min(orPrecedeIndex, -// andPrecedeIndex)).trim()); -// } else { -// filterByLogicalOperators.get(operator).add(filter.substring(index + MINIMUM_STEP_FROM_OPERATOR, Math.max(orPrecedeIndex, -// andPrecedeIndex)).trim()); -// } -//// filterByLogicalOperators.get(operator).add(filter.substring(index + MINIMUM_STEP_FROM_OPERATOR, andPrecedeIndex == -1 ? orPrecedeIndex : andPrecedeIndex).trim()); -// index = isAndOperator(operator) ? andPrecedeIndex : orPrecedeIndex; -// } else { -// filterByLogicalOperators.get(operator).add(filter.substring(index + MINIMUM_STEP_FROM_OPERATOR).trim()); -// index = isAndOperator(operator) ? andPrecedeIndex : orPrecedeIndex; -// } return index; } + private boolean isMatches(int index, int index1, String operator) { + return filter.substring(index, index1).matches(operator); + } + + private boolean isAndOrGreaterThanCurrentPosition(int index, int orPrecedeIndex, int andPrecedeIndex) { + return andPrecedeIndex > index && orPrecedeIndex > index; + } + + private boolean isEndOfFilter(int orPrecedeIndex, int andPrecedeIndex) { + return andPrecedeIndex == -1 && orPrecedeIndex == -1; + } + private boolean isLeftMostOR(boolean matches, int orPrecedeIndex, int andPrecedeIndex) { return orPrecedeIndex < andPrecedeIndex && matches; } From 6dd89191e2f6ce3e02f6b3b6170fe692d0ceb29b Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 29 Dec 2021 12:08:06 +0400 Subject: [PATCH 271/405] DDP-7121 | refactor AndOrFilterSeparator class to reorganize methods --- .../elastic/filter/AndOrFilterSeparator.java | 97 ++++++++----------- .../filter/AndOrFilterSeparatorTest.java | 13 +-- .../filter/CollectionQueryBuilderTest.java | 4 - 3 files changed, 45 insertions(+), 69 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java index 67aabcc0f..44149b964 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java @@ -1,13 +1,13 @@ package org.broadinstitute.dsm.model.elastic.filter; -import org.broadinstitute.dsm.model.Filter; - import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; +import org.broadinstitute.dsm.model.Filter; + public class AndOrFilterSeparator { public static final String DSM_ALIAS_REGEX = "(m|p|r|t|d|o|k|JS|ST|DA|\\()(\\.|\\s)*([a-z]|O|R|T|D|)(\\.)*"; @@ -44,16 +44,6 @@ protected Map> parseFiltersByLogicalOperators() { return filterByLogicalOperators; } - private void handleSpecialCases(Map> filterByLogicalOperators) { - final String additionalValuesJsonIsNotNull = "additional_values_json IS NOT NULL"; - for (Map.Entry> entry: filterByLogicalOperators.entrySet()) { - List filteredByNotAdditionalValuesIsNotNull = entry.getValue().stream() - .filter(f -> !f.contains(additionalValuesJsonIsNotNull)) - .collect(Collectors.toList()); - filterByLogicalOperators.put(entry.getKey(), filteredByNotAdditionalValuesIsNotNull); - } - } - /** * "AND ( oD.request = 'review' OR oD.request = 'no' ) " - filter * @@ -73,35 +63,6 @@ private int findProperOperatorSplitterIndex(String operator, int startIndex, int return startIndex; } - private boolean isOperatorWrappedInParenthesis(int startIndex) { - final char openParenthesis = '('; - final char closeParenthesis = ')'; - boolean exists = false; - for (int i = startIndex; i > 2; i--) { - char c = filter.charAt(i); - if (c == openParenthesis) { - exists = true; - break; - } - if (Filter.AND_TRIMMED.equals(filter.substring(i - 3, i)) || c == closeParenthesis) break; - // ) OR - } - return exists; - } - - private String getAliasRegexByOperator(String operator) { - return "AND".equals(operator) ? AND_DSM_ALIAS_REGEX : OR_DSM_ALIAS_REGEX; - } - - private int findNextOperatorIndex(String operator, int fromIndex) { - int index = filter.indexOf(operator, fromIndex); - if (isOperatorWrappedInParenthesis(index)) { - index = filter.indexOf(Filter.CLOSE_PARENTHESIS, index); - index = filter.indexOf(operator, index); - } - return index; - } - private int getIndex(Map> filterByLogicalOperators, int index, String operator) { int orPrecedeIndex = findNextOperatorIndex(Filter.OR_TRIMMED, index + MINIMUM_STEP_FROM_OPERATOR); @@ -128,36 +89,54 @@ private int getIndex(Map> filterByLogicalOperators, int ind return index; } - private boolean isMatches(int index, int index1, String operator) { - return filter.substring(index, index1).matches(operator); - } - - private boolean isAndOrGreaterThanCurrentPosition(int index, int orPrecedeIndex, int andPrecedeIndex) { - return andPrecedeIndex > index && orPrecedeIndex > index; + private void handleSpecialCases(Map> filterByLogicalOperators) { + final String additionalValuesJsonIsNotNull = "additional_values_json IS NOT NULL"; + for (Map.Entry> entry: filterByLogicalOperators.entrySet()) { + List filteredByNotAdditionalValuesIsNotNull = entry.getValue().stream() + .filter(f -> !f.contains(additionalValuesJsonIsNotNull)) + .collect(Collectors.toList()); + filterByLogicalOperators.put(entry.getKey(), filteredByNotAdditionalValuesIsNotNull); + } } - private boolean isEndOfFilter(int orPrecedeIndex, int andPrecedeIndex) { - return andPrecedeIndex == -1 && orPrecedeIndex == -1; + private String getAliasRegexByOperator(String operator) { + return Filter.AND_TRIMMED.equals(operator) ? AND_DSM_ALIAS_REGEX : OR_DSM_ALIAS_REGEX; } - private boolean isLeftMostOR(boolean matches, int orPrecedeIndex, int andPrecedeIndex) { - return orPrecedeIndex < andPrecedeIndex && matches; + private boolean isOperatorWrappedInParenthesis(int startIndex) { + final char openParenthesis = '('; + final char closeParenthesis = ')'; + boolean exists = false; + for (int i = startIndex; i > 2; i--) { + char c = filter.charAt(i); + if (c == openParenthesis) { + exists = true; + break; + } + if (Filter.AND_TRIMMED.equals(filter.substring(i - 3, i)) || c == closeParenthesis) break; + } + return exists; } - private boolean isLeftMostAND(boolean matches, int orPrecedeIndex, int andPrecedeIndex) { - return andPrecedeIndex < orPrecedeIndex && matches; + private int findNextOperatorIndex(String operator, int fromIndex) { + int index = filter.indexOf(operator, fromIndex); + if (isOperatorWrappedInParenthesis(index)) { + index = filter.indexOf(Filter.CLOSE_PARENTHESIS, index); + index = filter.indexOf(operator, index); + } + return index; } - private boolean isAndOperator(String operator) { - return "AND".equals(operator); + private boolean isMatches(int index, int index1, String operator) { + return filter.substring(index, index1).matches(operator); } - private int getPatternMatcherNumberByOperator(String operator) { - return isAndOperator(operator) ? AND_PATTERN_MATCHER_NUMBER : OR_PATTERN_MATCHER_NUMBER; + private boolean isAndOrGreaterThanCurrentPosition(int index, int orPrecedeIndex, int andPrecedeIndex) { + return andPrecedeIndex > index && orPrecedeIndex > index; } - private int getFilterIndex(int andIndex, int patternMatcherNumber) { - return andIndex + patternMatcherNumber; + private boolean isEndOfFilter(int orPrecedeIndex, int andPrecedeIndex) { + return andPrecedeIndex == -1 && orPrecedeIndex == -1; } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparatorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparatorTest.java index b06e22fb0..0a2981eaf 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparatorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparatorTest.java @@ -1,14 +1,11 @@ package org.broadinstitute.dsm.model.elastic.filter; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - import java.util.ArrayList; import java.util.List; import java.util.Map; -import static org.junit.Assert.*; +import org.junit.Assert; +import org.junit.Test; public class AndOrFilterSeparatorTest { @@ -22,10 +19,12 @@ public void parseFiltersByLogicalOperators() { "AND m.dynamicFields.ragac = '55' " + "AND ( t.tissue = 'review' OR t.tissue = 'no' OR t.tissue = 'bla' ) " + "OR m.medicalRecordName = '213' " + + "AND STR_TO_DATE(m.fax_sent,'%Y-%m-%d') = STR_TO_DATE('2021-12-17','%Y-%m-%d') " + "OR m.mrNotes = 'MEDICAL_RECORD_NOTESS' " + "AND m.medicalMedical = 'something AND something' " + "AND ( oD.request = 'review' OR oD.request = 'no' OR oD.request = 'bla' ) " + "OR t.tissueRecord = '225' " + + "OR STR_TO_DATE(m.fax_sent,'%Y-%m-%d') = STR_TO_DATE('2021-12-17','%Y-%m-%d') " + "AND JSON_EXTRACT ( m.additiona`l_values_json , '$.seeingIfBugExists' )"; AndOrFilterSeparator andOrFilterSeparator = new AndOrFilterSeparator(filter); Map> parsedFilters = andOrFilterSeparator.parseFiltersByLogicalOperators(); @@ -34,13 +33,15 @@ public void parseFiltersByLogicalOperators() { Assert.assertArrayEquals(new ArrayList<>(List.of("m.medicalRecordId = '15'", "t.tissueRecord IS NOT NULL" ,"m.dynamicFields.ragac = '55'", "( t.tissue = 'review' OR t.tissue = 'no' OR t.tissue = 'bla' )", + "STR_TO_DATE(m.fax_sent,'%Y-%m-%d') = STR_TO_DATE('2021-12-17','%Y-%m-%d')", "m.medicalMedical = 'something AND something'", "( oD.request = 'review' OR oD.request = 'no' OR " + "oD.request = 'bla' )", "JSON_EXTRACT ( m.additiona`l_values_json , '$.seeingIfBugExists' )")).toArray(), eachFilter.getValue().toArray()); } else { Assert.assertArrayEquals(new ArrayList<>(List.of("m.medicalRecordSomething LIKE '55555'", "m.medicalRecordSomethingg = " + - "'55552'", "m.medicalRecordName = '213'", "m.mrNotes = 'MEDICAL_RECORD_NOTESS'", "t.tissueRecord = '225'")).toArray(), + "'55552'", "m.medicalRecordName = '213'", "m.mrNotes = 'MEDICAL_RECORD_NOTESS'", "t.tissueRecord = '225'" + , "STR_TO_DATE(m.fax_sent,'%Y-%m-%d') = STR_TO_DATE('2021-12-17','%Y-%m-%d')")).toArray(), eachFilter.getValue().toArray()); } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java index 9ba6615b4..9fe075723 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java @@ -1,8 +1,5 @@ package org.broadinstitute.dsm.model.elastic.filter; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; import java.util.regex.Pattern; import org.apache.lucene.search.join.ScoreMode; @@ -14,7 +11,6 @@ import org.elasticsearch.index.query.RangeQueryBuilder; import org.junit.Assert; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; public class CollectionQueryBuilderTest { From d5efda5220b015c659106a6397688cab9534c9cd Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 29 Dec 2021 18:34:33 +0400 Subject: [PATCH 272/405] DDP-7121 | fix bugs related to serialization of dsm objects --- .../org/broadinstitute/dsm/db/MedicalRecord.java | 8 +++++--- src/main/java/org/broadinstitute/dsm/db/Tissue.java | 5 +++++ .../java/org/broadinstitute/dsm/db/ViewFilter.java | 4 ++++ .../java/org/broadinstitute/dsm/model/NameValue.java | 4 ++++ .../broadinstitute/dsm/model/ParticipantColumn.java | 6 ++++++ .../broadinstitute/dsm/model/filter/BaseFilter.java | 12 +++++++++++- .../broadinstitute/dsm/model/elastic/UtilTest.java | 3 +++ 7 files changed, 38 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java index 3be5357b2..d8a76889f 100644 --- a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java +++ b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java @@ -15,7 +15,6 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; import com.google.gson.annotations.SerializedName; import lombok.Data; @@ -292,6 +291,9 @@ public boolean isFollowUpRequired() { @ColumnName (DBConstants.FOLLOWUP_REQUIRED_TEXT) private String followUpRequiredText; + @JsonProperty("followupRequiredText") + public String getFollowUpRequiredText() { return followUpRequiredText; } + @TableName ( name = DBConstants.DDP_MEDICAL_RECORD, alias = DBConstants.DDP_MEDICAL_RECORD_ALIAS, @@ -322,7 +324,7 @@ public boolean isFollowUpRequired() { primaryKey = DBConstants.MEDICAL_RECORD_ID, columnPrefix = "") @ColumnName (DBConstants.NOTES) - private String mrNotes; + private String notes; @TableName ( name = DBConstants.DDP_MEDICAL_RECORD, @@ -407,7 +409,7 @@ public MedicalRecord(String medicalRecordId, String institutionId, String ddpIns this.international = international; this.crRequired = crRequired; this.pathologyPresent = pathologyPresent; - this.mrNotes = mrNotes; + this.notes = mrNotes; this.reviewMedicalRecord = reviewMedicalRecord; this.followUps = followUps; this.followUpRequired = followUpRequired; diff --git a/src/main/java/org/broadinstitute/dsm/db/Tissue.java b/src/main/java/org/broadinstitute/dsm/db/Tissue.java index 16a4dd68c..244275dd9 100644 --- a/src/main/java/org/broadinstitute/dsm/db/Tissue.java +++ b/src/main/java/org/broadinstitute/dsm/db/Tissue.java @@ -154,6 +154,11 @@ public Map getDynamicFields() { @ColumnName (DBConstants.H_E_COUNT) private Integer hECount; + @JsonProperty("hECount") + public Integer gethECount() { + return hECount; + } + public Tissue() {} public Tissue(String tissueId, String oncHistoryDetailId, String notes, Integer countReceived, String tissueType, diff --git a/src/main/java/org/broadinstitute/dsm/db/ViewFilter.java b/src/main/java/org/broadinstitute/dsm/db/ViewFilter.java index a16185f4a..cb6aa7e53 100644 --- a/src/main/java/org/broadinstitute/dsm/db/ViewFilter.java +++ b/src/main/java/org/broadinstitute/dsm/db/ViewFilter.java @@ -91,6 +91,10 @@ public class ViewFilter { public String parent; public String filterQuery; + public ViewFilter() { + + } + public ViewFilter(String filterName, String parent) { this(filterName, null, null, null, null, null, null, parent, null, null, null, null, null); diff --git a/src/main/java/org/broadinstitute/dsm/model/NameValue.java b/src/main/java/org/broadinstitute/dsm/model/NameValue.java index 2ef4187eb..3438aa460 100644 --- a/src/main/java/org/broadinstitute/dsm/model/NameValue.java +++ b/src/main/java/org/broadinstitute/dsm/model/NameValue.java @@ -12,4 +12,8 @@ public NameValue(String name, Object value) { this.name = name; this.value = value; } + + public NameValue() { + + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/ParticipantColumn.java b/src/main/java/org/broadinstitute/dsm/model/ParticipantColumn.java index 9c6fcb6b1..17a91a920 100644 --- a/src/main/java/org/broadinstitute/dsm/model/ParticipantColumn.java +++ b/src/main/java/org/broadinstitute/dsm/model/ParticipantColumn.java @@ -1,8 +1,10 @@ package org.broadinstitute.dsm.model; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.Getter; @Getter +@JsonIgnoreProperties(ignoreUnknown = true) public class ParticipantColumn { public String name; @@ -12,4 +14,8 @@ public ParticipantColumn(String name, String tableAlias) { this.name = name; this.tableAlias = tableAlias; } + + public ParticipantColumn() { + + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/filter/BaseFilter.java b/src/main/java/org/broadinstitute/dsm/model/filter/BaseFilter.java index 6212a6a44..87dc9ddad 100644 --- a/src/main/java/org/broadinstitute/dsm/model/filter/BaseFilter.java +++ b/src/main/java/org/broadinstitute/dsm/model/filter/BaseFilter.java @@ -1,7 +1,9 @@ package org.broadinstitute.dsm.model.filter; +import java.io.IOException; import java.util.Objects; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.DDPInstance; @@ -10,6 +12,7 @@ import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.statics.RequestParameter; import org.broadinstitute.dsm.statics.RoutePath; +import org.broadinstitute.dsm.util.ObjectMapperSingleton; import spark.QueryParamsMap; public class BaseFilter { @@ -42,7 +45,14 @@ protected void prepareNeccesaryData(QueryParamsMap queryParamsMap) { quickFilterName = ""; Filter[] savedFilters = new Gson().fromJson(queryParamsMap.get(RequestParameter.FILTERS).value(), Filter[].class); if (!Objects.isNull(jsonBody)) { - ViewFilter requestForFiltering = new Gson().fromJson(jsonBody, ViewFilter.class); + + ViewFilter requestForFiltering = null; + try { + requestForFiltering = ObjectMapperSingleton.instance().readValue(jsonBody, ViewFilter.class); + } catch (IOException e) { + throw new RuntimeException(e); + } + if (requestForFiltering != null) { if (requestForFiltering.getFilters() == null && StringUtils.isNotBlank(requestForFiltering.getFilterQuery())) { filterQuery = ViewFilter.changeFieldsInQuery(requestForFiltering.getFilterQuery(), false); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java index d842fb22d..e2e88445b 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java @@ -27,16 +27,19 @@ public void underscoresToCamelCase() { String fieldName3 = "column"; String fieldName4 = "COLUMN"; String fieldName5 = "columnName"; + String fieldName6 = "column.name"; String transformed = Util.underscoresToCamelCase(fieldName); String transformed2 = Util.underscoresToCamelCase(fieldName2); String transformed3 = Util.underscoresToCamelCase(fieldName3); String transformed4 = Util.underscoresToCamelCase(fieldName4); String transformed5 = Util.underscoresToCamelCase(fieldName5); + String transformed6 = Util.underscoresToCamelCase(fieldName6); assertEquals("columnName", transformed); assertEquals("columnName", transformed2); assertEquals("column", transformed3); assertEquals("column", transformed4); assertEquals("columnName", transformed5); + assertEquals("columnName", transformed6); } From b022f0eb74ab8ddc5ded96c2b651c967e1518bb6 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 30 Dec 2021 11:29:10 +0400 Subject: [PATCH 273/405] DDP-7121 | add JsonGetter to help Jackson to serialize values properly if field has Opional type getter --- .../ddp/participant/ParticipantDataDto.java | 22 +++++++++++++++++++ .../dsm/model/filter/BaseFilter.java | 4 +++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java b/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java index 1d43b23ee..363aa7b61 100644 --- a/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java +++ b/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java @@ -6,8 +6,10 @@ import java.util.Map; import java.util.Optional; +import com.fasterxml.jackson.annotation.JsonGetter; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonRawValue; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; @@ -38,12 +40,32 @@ public class ParticipantDataDto { @ColumnName(ParticipantDataDao.DDP_PARTICIPANT_ID) private String ddpParticipantId; + /* + used only for Jackson library, jackson by default uses getter of the field to serialize its data + since we follow Optional way of getters, jackson by default weirdly serializes Optional and not field's data + JsonGetter is used to give a hint to Jackson to use specific method to serialize data for the field + */ + @JsonGetter("ddpParticipantId") + private String serializeDdpParticipantId() { + return ddpParticipantId; + } + @ColumnName(ParticipantDataDao.DDP_INSTANCE_ID) private int ddpInstanceId; @ColumnName(ParticipantDataDao.FIELD_TYPE_ID) private String fieldTypeId; + /* + used only for Jackson library, jackson by default uses getter of the field to serialize its data + since we follow Optional way of getters, jackson by default weirdly serializes Optional and not field's data + JsonGetter is used to give a hint to Jackson to use specific method to serialize data for the field + */ + @JsonGetter("fieldTypeId") + private String serializeFieldTypeId() { + return fieldTypeId; + } + @ColumnName(ParticipantDataDao.DATA) @JsonProperty("dynamicFields") private String data; diff --git a/src/main/java/org/broadinstitute/dsm/model/filter/BaseFilter.java b/src/main/java/org/broadinstitute/dsm/model/filter/BaseFilter.java index 87dc9ddad..38bf8c57d 100644 --- a/src/main/java/org/broadinstitute/dsm/model/filter/BaseFilter.java +++ b/src/main/java/org/broadinstitute/dsm/model/filter/BaseFilter.java @@ -48,7 +48,9 @@ protected void prepareNeccesaryData(QueryParamsMap queryParamsMap) { ViewFilter requestForFiltering = null; try { - requestForFiltering = ObjectMapperSingleton.instance().readValue(jsonBody, ViewFilter.class); + requestForFiltering = StringUtils.isNotBlank(jsonBody) + ? ObjectMapperSingleton.instance().readValue(jsonBody, ViewFilter.class) + : null; } catch (IOException e) { throw new RuntimeException(e); } From 553f5738c052aea9e6d17b4c8173f50c8e1c15a0 Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 30 Dec 2021 12:24:57 +0400 Subject: [PATCH 274/405] DDP-7121 | ignore some fields for ParticipantDataDto while serializing its objects --- .../ddp/participant/ParticipantDataDto.java | 9 +++-- .../dsm/model/elastic/UtilTest.java | 15 +++++++- .../DynamicFieldsMappingMigratorTest.java | 4 +- .../search/SourceMapDeserializerTest.java | 37 ++++++++++++++++++- 4 files changed, 56 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java b/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java index 363aa7b61..8ffd1d72a 100644 --- a/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java +++ b/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java @@ -6,10 +6,7 @@ import java.util.Map; import java.util.Optional; -import com.fasterxml.jackson.annotation.JsonGetter; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonRawValue; +import com.fasterxml.jackson.annotation.*; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; @@ -81,11 +78,14 @@ public Map getDynamicFields() { public ParticipantDataDto() {} + @JsonIgnore private long lastChanged; + @JsonIgnore private String changedBy; // We cache the json data map to avoid deserializing it multiple times. + @JsonIgnore private Map cachedDataMap; public int getParticipantDataId() { @@ -113,6 +113,7 @@ public void setData(String data) { this.cachedDataMap = null; } + @JsonIgnore public Map getDataMap() { if (cachedDataMap != null) { return cachedDataMap; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java index e2e88445b..c0e58e2c9 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java @@ -118,11 +118,22 @@ class MockClass { @Test public void camelCaseToPascalSnakeCase() { String camelCase = "registrationType"; - String camelCase2 = "test"; + String camelCaseVal = "test"; + + String camelCase2 = "medicalRecordsReleaseObtained"; + String camelCase2Val = "NO"; + String pascalSnakeCase = Util.camelCaseToPascalSnakeCase(camelCase); + String pascalSnakeCaseVal = Util.camelCaseToPascalSnakeCase(camelCaseVal); + String pascalSnakeCase2 = Util.camelCaseToPascalSnakeCase(camelCase2); + String pascalSnakeCase2Val = Util.camelCaseToPascalSnakeCase(camelCase2Val); + assertEquals("REGISTRATION_TYPE", pascalSnakeCase); - assertEquals("TEST", pascalSnakeCase2); + assertEquals("TEST", pascalSnakeCaseVal); + + assertEquals("MEDICAL_RECORDS_RELEASE_OBTAINED", pascalSnakeCase2); + assertEquals("NO", pascalSnakeCase2Val); } @Test diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java index 53e7a9928..30aab1fe9 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java @@ -17,8 +17,8 @@ public static void setUp() { @Test public void testExport() { - final String index = "participants_structured.cmi.angio"; - final String study = "angio"; + final String index = "participants_structured.rgp.rgp"; + final String study = "rgp"; List exportables = Arrays.asList( //DynamicFieldsMappingMigrator should be first in the list to make sure that mapping will be exported for first new DynamicFieldsMappingMigrator(index, study), diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializerTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializerTest.java index 4b00faa35..3100028da 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializerTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializerTest.java @@ -3,7 +3,7 @@ import static org.junit.Assert.*; import java.io.IOException; -import java.util.Map; +import java.util.*; import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ObjectMapperSingleton; @@ -23,6 +23,8 @@ public void convertFollowUpsJsonToList() { "ddpInstanceId", 12, ESObjectConstants.DYNAMIC_FIELDS, dynamicFields ); + + SourceMapDeserializer sourceMapDeserializer = new SourceMapDeserializer(); sourceMapDeserializer.outerProperty = ESObjectConstants.PARTICIPANT_DATA; try { @@ -34,4 +36,37 @@ public void convertFollowUpsJsonToList() { Assert.fail(); } } + + @Test + public void convertSeveralDynamicFields() { + + Map dynamicFields1 = new HashMap<>(Map.of( + "registrationType", "Self", + "registrationStatus", "Registered" + )); + + Map dynamicFields2 = new HashMap<>(Map.of( + "registrationType", "Self", + "registrationStatus", "Registered" + )); + + Map outerProperties1 = new HashMap<>(Map.of( + "ddpInstanceId", 12, + ESObjectConstants.DYNAMIC_FIELDS, dynamicFields1 + )); + + Map outerProperties2 = new HashMap<>(Map.of( + "ddpInstanceId", 13, + ESObjectConstants.DYNAMIC_FIELDS, dynamicFields2 + )); + + Map participantData = new HashMap<>(Map.of("participantData", new ArrayList<>(Arrays.asList(outerProperties1, outerProperties2)))); + Map dsm = new HashMap<>(Map.of("dsm", participantData)); + + Optional des = new SourceMapDeserializer().deserialize(dsm); + + System.out.println(des); + + + } } \ No newline at end of file From ead0ebda1548132f623e04db8f862088a7fd3eca Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 30 Dec 2021 12:59:31 +0400 Subject: [PATCH 275/405] DDP-7121 | add default deserializer to avoid deserialize dynamic field fields for exporting --- .../model/elastic/export/ExportFacade.java | 2 ++ .../elastic/search/DefaultDeserializer.java | 14 ++++++++++ .../model/elastic/search/ElasticSearch.java | 28 ++++++++++++------- .../elastic/search/ElasticSearchable.java | 4 +++ .../dsm/util/ElasticSearchUtil.java | 5 ++-- .../elastic/search/ElasticSearchTest.java | 2 +- 6 files changed, 42 insertions(+), 13 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/search/DefaultDeserializer.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 2532d0dbd..b6f5f4400 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -20,6 +20,7 @@ import org.broadinstitute.dsm.model.elastic.export.process.ProcessorFactory; import org.broadinstitute.dsm.model.elastic.export.process.ProcessorFactoryImpl; import org.broadinstitute.dsm.model.elastic.export.process.SingleProcessor; +import org.broadinstitute.dsm.model.elastic.search.DefaultDeserializer; import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchable; @@ -65,6 +66,7 @@ private void upsertMapping() { } private ESDsm fetchData() { + searchable.setDeserializer(new DefaultDeserializer()); ElasticSearchParticipantDto participantById = searchable.getParticipantById(exportFacadePayload.getIndex(), exportFacadePayload.getDocId()); // Ensure that participant data will be stored by participant guid exportFacadePayload.setDocId(participantById.getParticipantId()); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/DefaultDeserializer.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/DefaultDeserializer.java new file mode 100644 index 000000000..b105c6094 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/DefaultDeserializer.java @@ -0,0 +1,14 @@ +package org.broadinstitute.dsm.model.elastic.search; + +import java.util.Map; +import java.util.Optional; + +import org.broadinstitute.dsm.util.ObjectMapperSingleton; + +public class DefaultDeserializer implements Deserializer { + + @Override + public Optional deserialize(Map sourceMap) { + return Optional.ofNullable(ObjectMapperSingleton.instance().convertValue(sourceMap, ElasticSearchParticipantDto.class)); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java index 2b3afcd0c..13b7b523b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearch.java @@ -36,32 +36,40 @@ public class ElasticSearch implements ElasticSearchable { private static final Logger logger = LoggerFactory.getLogger(ElasticSearch.class); - private static final Gson GSON = new Gson(); + private Deserializer deserializer; List esParticipants; long totalCount; - public List getEsParticipants() { - if (Objects.isNull(esParticipants)) { - esParticipants = Collections.emptyList(); - } - return esParticipants; - } - public ElasticSearch() {} + public ElasticSearch() { + this.deserializer = new SourceMapDeserializer(); + } public ElasticSearch(List esParticipants, long totalCount) { + this(); this.esParticipants = esParticipants; this.totalCount = totalCount; } + public void setDeserializer(Deserializer deserializer) { + this.deserializer = deserializer; + } + + public List getEsParticipants() { + if (Objects.isNull(esParticipants)) { + esParticipants = Collections.emptyList(); + } + return esParticipants; + } + public long getTotalCount() { return totalCount; } - public static Optional parseSourceMap(Map sourceMap) { + public Optional parseSourceMap(Map sourceMap) { if (sourceMap == null) return Optional.of(new ElasticSearchParticipantDto.Builder().build()); - Optional deserializedSourceMap = new SourceMapDeserializer().deserialize(sourceMap); + Optional deserializedSourceMap = deserializer.deserialize(sourceMap); return deserializedSourceMap.isPresent() ? deserializedSourceMap : Optional.of(new ElasticSearchParticipantDto.Builder().build()); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchable.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchable.java index fb09f45d7..7515872cf 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchable.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchable.java @@ -21,4 +21,8 @@ public interface ElasticSearchable { ElasticSearch getAllParticipantsDataByInstanceIndex(String esParticipantsIndex); + default void setDeserializer(Deserializer deserializer) { + + } + } diff --git a/src/main/java/org/broadinstitute/dsm/util/ElasticSearchUtil.java b/src/main/java/org/broadinstitute/dsm/util/ElasticSearchUtil.java index 503df435f..b65a2b780 100644 --- a/src/main/java/org/broadinstitute/dsm/util/ElasticSearchUtil.java +++ b/src/main/java/org/broadinstitute/dsm/util/ElasticSearchUtil.java @@ -308,7 +308,8 @@ public static ElasticSearchParticipantDto getElasticSearchForGivenMatch(String i response = client.search(searchRequest, RequestOptions.DEFAULT); response.getHits(); - return ElasticSearch.parseSourceMap(response.getHits().getTotalHits() > 0 ? response.getHits().getAt(0).getSourceAsMap() : null).get(); + ElasticSearch elasticSearch = new ElasticSearch(); + return elasticSearch.parseSourceMap(response.getHits().getTotalHits() > 0 ? response.getHits().getAt(0).getSourceAsMap() : null).get(); } public static Map> getDDPParticipantsFromES(@NonNull String realm, @NonNull String index) { @@ -745,7 +746,7 @@ public static Optional getParticipantProfileByGuidOrAltPid(String ind ESProfile profile = null; if (response.getHits().getTotalHits() > 0) { Map source = response.getHits().getAt(0).getSourceAsMap(); - profile = ElasticSearch.parseSourceMap(source).flatMap(ElasticSearchParticipantDto::getProfile).orElse(null); + profile = new ElasticSearch().parseSourceMap(source).flatMap(ElasticSearchParticipantDto::getProfile).orElse(null); if (profile != null) { logger.info("Found ES profile for participant, guid: {} altpid: {}", profile.getGuid(), profile.getLegacyAltPid()); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchTest.java index 845aa48ee..1d120334e 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchTest.java @@ -74,7 +74,7 @@ public void parseSourceMap() { .withAddress(esAddress) .build(); Map esMap = GSON.fromJson(GSON.toJson(elasticSearchParticipantDto), Map.class); - Optional maybeElasticSearchParticipantDto = ElasticSearch.parseSourceMap(esMap); + Optional maybeElasticSearchParticipantDto = new ElasticSearch().parseSourceMap(esMap); try { ElasticSearchParticipantDto esParticipantDto = maybeElasticSearchParticipantDto.get(); Assert.assertEquals("Tommy", esParticipantDto.getProfile().map(ESProfile::getFirstName).orElse("")); From d42de72a300ff176d4216f8304e51db6f536365d Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 30 Dec 2021 15:35:45 +0400 Subject: [PATCH 276/405] DDP-7121 | introduce default deserializer if we don't want at some point to convert dynamic fields from camel to pascal case --- .../elastic/search/DefaultDeserializer.java | 9 ++++----- .../elastic/search/SourceMapDeserializer.java | 16 ++++++++++------ 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/DefaultDeserializer.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/DefaultDeserializer.java index b105c6094..003e3b688 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/DefaultDeserializer.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/DefaultDeserializer.java @@ -1,14 +1,13 @@ package org.broadinstitute.dsm.model.elastic.search; import java.util.Map; -import java.util.Optional; -import org.broadinstitute.dsm.util.ObjectMapperSingleton; +public class DefaultDeserializer extends SourceMapDeserializer { -public class DefaultDeserializer implements Deserializer { + //used if we don't need to convert dynamic fields to pascal case @Override - public Optional deserialize(Map sourceMap) { - return Optional.ofNullable(ObjectMapperSingleton.instance().convertValue(sourceMap, ElasticSearchParticipantDto.class)); + protected Map convertDynamicFieldsFromCamelCaseToPascalCase(Map dynamicFields) { + return dynamicFields; } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java index 4136e8624..d52d5be7a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java @@ -12,7 +12,6 @@ import java.io.IOException; import java.lang.reflect.Field; import java.util.*; -import java.util.concurrent.ConcurrentHashMap; public class SourceMapDeserializer implements Deserializer { @@ -82,11 +81,7 @@ private List> convertFollowUpsJsonToList(Map String getDynamicFieldsValueAsJson(Map clonedMap) { Map dynamicFields = (Map) clonedMap.get(ESObjectConstants.DYNAMIC_FIELDS); if (ESObjectConstants.PARTICIPANT_DATA.equals(outerProperty)) { - Map updatedParticipantDataDynamicFields = new HashMap<>(); - for (Map.Entry entry: dynamicFields.entrySet()) { - updatedParticipantDataDynamicFields.put(Util.camelCaseToPascalSnakeCase(entry.getKey()), entry.getValue()); - } - dynamicFields = updatedParticipantDataDynamicFields; + dynamicFields = convertDynamicFieldsFromCamelCaseToPascalCase(dynamicFields); } try { return dynamicFields.isEmpty() @@ -97,6 +92,15 @@ String getDynamicFieldsValueAsJson(Map clonedMap) { } } + protected Map convertDynamicFieldsFromCamelCaseToPascalCase(Map dynamicFields) { + Map updatedParticipantDataDynamicFields = new HashMap<>(); + for (Map.Entry entry: dynamicFields.entrySet()) { + updatedParticipantDataDynamicFields.put(Util.camelCaseToPascalSnakeCase(entry.getKey()), entry.getValue()); + } + dynamicFields = updatedParticipantDataDynamicFields; + return dynamicFields; + } + private boolean hasDynamicFields(String outerProperty) { try { Field property = ESDsm.class.getDeclaredField(outerProperty); From e1076c0c67d2b2ac793ae4d4bddcbaa077d15efc Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 30 Dec 2021 17:24:12 +0400 Subject: [PATCH 277/405] DDP-7121 | comment out ond code regarding "ES" filtering --- .../BaseFilterParticipantList.java | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java b/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java index 62ee7b1d9..67fc91e28 100644 --- a/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java +++ b/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java @@ -80,24 +80,27 @@ protected ParticipantWrapperResult filterParticipantList(Filter[] filters, Map Date: Mon, 3 Jan 2022 11:19:40 +0400 Subject: [PATCH 278/405] DDP-7121 | cleanup code --- .../BaseFilterParticipantList.java | 30 ++++---- .../model/participant/ParticipantWrapper.java | 72 ------------------- 2 files changed, 15 insertions(+), 87 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java b/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java index 67fc91e28..8fbf04d45 100644 --- a/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java +++ b/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java @@ -80,27 +80,27 @@ protected ParticipantWrapperResult filterParticipantList(Filter[] filters, Map filters) { - List participantIdsToFetch = Collections.emptyList(); FilterParser parser = new FilterParser(); BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); for (String source : filters.keySet()) { @@ -98,28 +97,8 @@ private void fetchAndPrepareDataByFilters(DDPInstance ddpInstance, Map participants = -// Participant.getParticipants(ddpInstance.getName(), filters.get(source)); -// this.participants = new ArrayList<>(participants.values()); -// participantIdsToFetch = new ArrayList<>(participants.keySet()); -// } -// else if (DBConstants.DDP_MEDICAL_RECORD_ALIAS.equals(source)) { -// medicalRecords = MedicalRecord.getMedicalRecords(ddpInstance.getName(), filters.get(source)); -// participantIdsToFetch = new ArrayList<>(medicalRecords.keySet()); -// } -// else if (DBConstants.DDP_ONC_HISTORY_DETAIL_ALIAS.equals(source)) { -// oncHistoryDetails = OncHistoryDetail.getOncHistoryDetails(ddpInstance.getName(), filters.get(source)); -// participantIdsToFetch = new ArrayList<>(oncHistoryDetails.keySet()); -// } -// else if (DBConstants.DDP_KIT_REQUEST_ALIAS.equals(source)) { -// kitRequests = KitRequestShipping.getKitRequests(ddpInstance, filters.get(source)); -// participantIdsToFetch = new ArrayList<>(kitRequests.keySet()); -// } - else if (DBConstants.DDP_PARTICIPANT_DATA_ALIAS.equals(source)) { participantData = new ParticipantDataDao().getParticipantDataByInstanceIdAndFilterQuery(Integer.parseInt(ddpInstance.getDdpInstanceId()), filters.get(source)); - participantIdsToFetch = new ArrayList<>(participantData.keySet()); //if study is AT TODO if ("atcp".equals(ddpInstance.getName())) { @@ -128,16 +107,7 @@ else if (DBConstants.DDP_PARTICIPANT_DATA_ALIAS.equals(source)) { participantData = defaultValues.addDefaultValues(); } } -// else if (DBConstants.DDP_ABSTRACTION_ALIAS.equals(source)) { -// abstractionActivities = AbstractionActivity.getAllAbstractionActivityByRealm(ddpInstance.getName(), filters.get(source)); -// participantIdsToFetch = new ArrayList<>(abstractionActivities.keySet()); -// } else if ("ES".equals(source)){ //source is not of any study-manager table so it must be ES -// esData = elasticSearchable.getParticipantsByRangeAndFilter(ddpInstance.getParticipantIndexES(), participantWrapperPayload.getFrom(), -// participantWrapperPayload.getTo(), boolQueryBuilder); -// participantIdsToFetch = esData.getEsParticipants().stream().map(ElasticSearchParticipantDto::getParticipantId) -// .collect( -// Collectors.toList()); boolQueryBuilder.must(ElasticSearchUtil.createESQuery(filters.get(source))); } } @@ -146,48 +116,6 @@ else if ("ES".equals(source)){ //source is not of any study-manager table so it esData = elasticSearchable.getParticipantsByRangeAndFilter(ddpInstance.getParticipantIndexES(), participantWrapperPayload.getFrom(), participantWrapperPayload.getTo(), boolQueryBuilder); -// if (esData.getEsParticipants().isEmpty()) { -// esData = elasticSearchable.getParticipantsByRangeAndIds(ddpInstance.getParticipantIndexES(), participantWrapperPayload.getFrom(), -// participantWrapperPayload.getTo(), participantIdsToFetch); -// } -// if (participants.isEmpty()) { -// participants = Participant.getParticipantsByIds(ddpInstance.getName(), participantIdsToFetch); -// } -// if (medicalRecords.isEmpty() && ddpInstance.isHasRole()) { -// medicalRecords = MedicalRecord.getMedicalRecordsByParticipantIds(ddpInstance.getName(), participantIdsToFetch); -// } -// if (oncHistoryDetails.isEmpty() && ddpInstance.isHasRole()) { -// oncHistoryDetails = OncHistoryDetail.getOncHistoryDetailsByParticipantIds(ddpInstance.getName(), participantIdsToFetch); -// } -// if (kitRequests.isEmpty() && DDPInstanceDao.getRole(ddpInstance.getName(), DBConstants.KIT_REQUEST_ACTIVATED)) { //only needed if study is shipping samples per DSM -// //get only kitRequests for the filtered pts -// if (Objects.nonNull(esData) && !esData.getEsParticipants().isEmpty()) { -// logger.info("About to query for kits from " + esData.getEsParticipants().size() + " participants"); -// kitRequests = KitRequestShipping.getKitRequestsByParticipantIds(ddpInstance, participantIdsToFetch); -// } -// } - -// if (participantData.isEmpty()) { -// participantData = new ParticipantDataDao().getParticipantDataByParticipantIds(participantIdsToFetch); -// -// // if study is AT -// // TODO - what's the purpose of this if statement -// if ("atcp".equals(ddpInstance.getName())) { -// DefaultValues defaultValues = new DefaultValues(participantData, esData.getEsParticipants(), ddpInstance, null); -// participantData = defaultValues.addDefaultValues(); -// } -// } -// if (abstractionActivities.isEmpty()) { -// abstractionActivities = AbstractionActivity.getAllAbstractionActivityByRealm(ddpInstance.getName()); -// } -// if (abstractionSummary.isEmpty()) { -// abstractionSummary = AbstractionFinal.getAbstractionFinal(ddpInstance.getName()); -// } -// if (proxiesByParticipantIds.isEmpty()) { -// proxiesByParticipantIds = getProxiesWithParticipantIdsFromElasticList(ddpInstance.getUsersIndexES(), esData.getEsParticipants()); -// } - - System.out.println(); System.out.println(); } From 97ddf3e7e279e0b059992f73e8f25ee73ec28e85 Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 3 Jan 2022 15:10:46 +0400 Subject: [PATCH 279/405] DDP-7121 | remove comments, refactor the code little bit --- .../BaseFilterParticipantList.java | 16 +------- .../model/participant/ParticipantWrapper.java | 37 +++++++------------ 2 files changed, 14 insertions(+), 39 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java b/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java index 8fbf04d45..e9be12ccd 100644 --- a/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java +++ b/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java @@ -104,19 +104,9 @@ protected ParticipantWrapperResult filterParticipantList(Filter[] filters, Map mergeConditions = new HashMap<>(); for (String filter : queryConditions.keySet()) { -// if (DBConstants.DDP_PARTICIPANT_RECORD_ALIAS.equals(filter) -// || DBConstants.DDP_PARTICIPANT_EXIT_ALIAS.equals(filter) || DBConstants.DDP_ONC_HISTORY_ALIAS.equals(filter)) { -// mergeConditions.merge(DBConstants.DDP_PARTICIPANT_ALIAS, queryConditions.get(filter), String::concat); -// } else if (DBConstants.DDP_INSTITUTION_ALIAS.equals(filter)) { -// mergeConditions.merge(DBConstants.DDP_MEDICAL_RECORD_ALIAS, queryConditions.get(filter), String::concat); -// } else if (DBConstants.DDP_TISSUE_ALIAS.equals(filter)) { -// mergeConditions.merge(DBConstants.DDP_ONC_HISTORY_DETAIL_ALIAS, queryConditions.get(filter), String::concat); -// } else { -// mergeConditions.merge(filter, queryConditions.get(filter), String::concat); -// } if (DBConstants.DDP_PARTICIPANT_EXIT_ALIAS.equals(filter)) { String exitFilter = queryConditions.get(filter).replace("ex.", "p."); mergeConditions.merge(DBConstants.DDP_PARTICIPANT_ALIAS, exitFilter, String::concat); @@ -124,11 +114,7 @@ protected ParticipantWrapperResult filterParticipantList(Filter[] filters, Map instead of passing mergeConditions passing queryConditions directly to avoid merge different keys like o,r -> p and - // etc. return new ParticipantWrapper(participantWrapperPayload.withFilter(mergeConditions).build(), elasticSearch).getFilteredList(); } else { return new ParticipantWrapper(participantWrapperPayload.build(), elasticSearch).getFilteredList(); diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java index be44af058..004ad6c3b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java @@ -96,7 +96,6 @@ private void fetchAndPrepareDataByFilters(DDPInstance ddpInstance, Map collectData(DDPInstance ddpInstance) { List oncHistoryDetails = esDsm.getOncHistoryDetail(); List tissues = esDsm.getTissue(); - for (Tissue tissue : tissues) { - String oncHistoryDetailId = tissue.getOncHistoryDetailId(); - oncHistoryDetails.stream() - .filter(oncHistoryDetail -> oncHistoryDetail.getOncHistoryDetailId().equals(oncHistoryDetailId)) - .findFirst() - .ifPresent(oncHistoryDetail -> oncHistoryDetail.getTissues().add(tissue)); - } + mapTissueToProperOncHistoryDetail(oncHistoryDetails, tissues); List kitRequestShipping = esDsm.getKitRequestShipping(); @@ -251,7 +241,6 @@ private List collectData(DDPInstance ddpInstance) { List participantData = esDsm.getParticipantData(); - ParticipantWrapperDto participantWrapperDto = new ParticipantWrapperDto(); participantWrapperDto.setEsData(elasticSearchParticipantDto); participantWrapperDto.setParticipant(participant); @@ -266,20 +255,20 @@ private List collectData(DDPInstance ddpInstance) { result.add(participantWrapperDto); }); -// String participantId = elasticSearchParticipantDto.getParticipantId(); -// if (StringUtils.isBlank(participantId)) continue; -// Participant participant = participants.stream() -// .filter(ppt -> participantId.equals(ppt.getDdpParticipantId())) -// .findFirst() -// .orElse(null); -// result.add(new ParticipantWrapperDto( -// elasticSearchParticipantDto, participant, medicalRecords.get(participantId), -// oncHistoryDetails.get(participantId), kitRequests.get(participantId), abstractionActivities.get(participantId), -// abstractionSummary.get(participantId), proxiesByParticipantIds.get(participantId), participantData.get(participantId))); } return result; } + private void mapTissueToProperOncHistoryDetail(List oncHistoryDetails, List tissues) { + for (Tissue tissue : tissues) { + String oncHistoryDetailId = tissue.getOncHistoryDetailId(); + oncHistoryDetails.stream() + .filter(oncHistoryDetail -> oncHistoryDetail.getOncHistoryDetailId().equals(oncHistoryDetailId)) + .findFirst() + .ifPresent(oncHistoryDetail -> oncHistoryDetail.getTissues().add(tissue)); + } + } + void sortBySelfElseById(Collection> participantDatas) { participantDatas.forEach(pDataList -> pDataList.sort((o1, o2) -> { Map pData = new Gson().fromJson(o1.getData().orElse(""), new TypeToken>() {}.getType()); From 5af32b2c16e8fe9e35a78d3716d2c05a8776af80 Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 3 Jan 2022 15:11:24 +0400 Subject: [PATCH 280/405] DDP-7121 | start changing the field data types for those classes which are involved in data migration and indexing in ES --- .../broadinstitute/dsm/db/MedicalRecord.java | 22 ++++---- .../dsm/db/OncHistoryDetail.java | 12 ++--- .../dsm/db/ParticipantData.java | 8 +-- .../org/broadinstitute/dsm/db/Tissue.java | 34 ++++++------- .../broadinstitute/dsm/model/TissueList.java | 2 +- .../dsm/model/at/SearchKitRequest.java | 2 +- .../model/participant/ParticipantWrapper.java | 4 +- .../dsm/model/elastic/UtilTest.java | 50 ++++--------------- .../migration/BaseCollectionMigratorTest.java | 8 +-- 9 files changed, 56 insertions(+), 86 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java index d8a76889f..c820ad6fa 100644 --- a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java +++ b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java @@ -69,7 +69,7 @@ public class MedicalRecord { primaryKey = DBConstants.MEDICAL_RECORD_ID, columnPrefix = "") @ColumnName (DBConstants.MEDICAL_RECORD_ID) - private String medicalRecordId; + private long medicalRecordId; @TableName ( name = DBConstants.DDP_INSTITUTION, @@ -77,7 +77,7 @@ public class MedicalRecord { primaryKey = DBConstants.INSTITUTION_ID, columnPrefix = "") @ColumnName (DBConstants.INSTITUTION_ID) - private String institutionId; + private long institutionId; @TableName ( name = DBConstants.DDP_INSTITUTION, @@ -85,7 +85,7 @@ public class MedicalRecord { primaryKey = DBConstants.DDP_INSTITUTION_ID, columnPrefix = "") @ColumnName (DBConstants.DDP_INSTITUTION_ID) - private String ddpInstitutionId; + private long ddpInstitutionId; @TableName ( name = DBConstants.DDP_PARTICIPANT, @@ -363,7 +363,7 @@ public Map getDynamicFields() { @ColumnName (DBConstants.PATHOLOGY_PRESENT) private String pathologyPresent; - public MedicalRecord(String medicalRecordId, String institutionId, String ddpInstitutionId, String type) { + public MedicalRecord(long medicalRecordId, long institutionId, long ddpInstitutionId, String type) { this.medicalRecordId = medicalRecordId; this.institutionId = institutionId; this.ddpInstitutionId = ddpInstitutionId; @@ -372,7 +372,7 @@ public MedicalRecord(String medicalRecordId, String institutionId, String ddpIns public MedicalRecord() {} - public MedicalRecord(String medicalRecordId, String institutionId, String ddpInstitutionId, String type, + public MedicalRecord(long medicalRecordId, long institutionId, long ddpInstitutionId, String type, String name, String contact, String phone, String fax, String faxSent, String faxSentBy, String faxConfirmed, String faxSent2, String faxSent2By, String faxConfirmed2, @@ -421,9 +421,9 @@ public MedicalRecord(String medicalRecordId, String institutionId, String ddpIns public static MedicalRecord getMedicalRecord(@NonNull ResultSet rs) throws SQLException { MedicalRecord medicalRecord = new MedicalRecord( - rs.getString(DBConstants.MEDICAL_RECORD_ID), - rs.getString(DBConstants.INSTITUTION_ID), - rs.getString(DBConstants.DDP_INSTITUTION_ID), + rs.getLong(DBConstants.MEDICAL_RECORD_ID), + rs.getLong(DBConstants.INSTITUTION_ID), + rs.getLong(DBConstants.DDP_INSTITUTION_ID), rs.getString(DBConstants.TYPE), rs.getString(DBConstants.NAME), rs.getString(DBConstants.CONTACT), @@ -550,9 +550,9 @@ public static List getInstitutions(@NonNull String realm, @NonNul try (ResultSet rs = stmt.executeQuery()) { while (rs.next()) { medicalRecords.add(new MedicalRecord( - rs.getString(DBConstants.MEDICAL_RECORD_ID), - rs.getString(DBConstants.INSTITUTION_ID), - rs.getString(DBConstants.DDP_INSTITUTION_ID), + rs.getLong(DBConstants.MEDICAL_RECORD_ID), + rs.getLong(DBConstants.INSTITUTION_ID), + rs.getLong(DBConstants.DDP_INSTITUTION_ID), rs.getString(DBConstants.TYPE))); } } diff --git a/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java b/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java index 02039fa09..b65d958c7 100644 --- a/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java +++ b/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java @@ -87,10 +87,10 @@ public class OncHistoryDetail { public static final String PROBLEM_OTHER_OLD = "Other"; @ColumnName(DBConstants.ONC_HISTORY_DETAIL_ID) - private String oncHistoryDetailId; + private long oncHistoryDetailId; @ColumnName(DBConstants.MEDICAL_RECORD_ID) - private String medicalRecordId; + private long medicalRecordId; @ColumnName (DBConstants.DATE_PX) private String datePx; @@ -196,7 +196,7 @@ public Map getDynamicFields() { public OncHistoryDetail() {} - public OncHistoryDetail(String oncHistoryDetailId, String medicalRecordId, String datePx, String typePx, + public OncHistoryDetail(long oncHistoryDetailId, long medicalRecordId, String datePx, String typePx, String locationPx, String histology, String accessionNumber, String facility, String phone, String fax, String notes, String request, String faxSent, String faxSentBy, String faxConfirmed, @@ -234,7 +234,7 @@ public OncHistoryDetail(String oncHistoryDetailId, String medicalRecordId, Strin this.unableObtainTissue = unableObtainTissue; } - public OncHistoryDetail(String oncHistoryDetailId, String medicalRecordId, String datePx, String typePx, + public OncHistoryDetail(long oncHistoryDetailId, long medicalRecordId, String datePx, String typePx, String locationPx, String histology, String accessionNumber, String facility, String phone, String fax, String notes, String request, String faxSent, String faxSentBy, String faxConfirmed, @@ -275,8 +275,8 @@ public OncHistoryDetail(String oncHistoryDetailId, String medicalRecordId, Strin public static OncHistoryDetail getOncHistoryDetail(@NonNull ResultSet rs) throws SQLException { List tissues = new ArrayList<>(); OncHistoryDetail oncHistoryDetail = new OncHistoryDetail( - rs.getString(DBConstants.ONC_HISTORY_DETAIL_ID), - rs.getString(DBConstants.MEDICAL_RECORD_ID), + rs.getLong(DBConstants.ONC_HISTORY_DETAIL_ID), + rs.getLong(DBConstants.MEDICAL_RECORD_ID), rs.getString(DBConstants.DATE_PX), rs.getString(DBConstants.TYPE_PX), rs.getString(DBConstants.LOCATION_PX), diff --git a/src/main/java/org/broadinstitute/dsm/db/ParticipantData.java b/src/main/java/org/broadinstitute/dsm/db/ParticipantData.java index 976515c1a..945fbd59d 100644 --- a/src/main/java/org/broadinstitute/dsm/db/ParticipantData.java +++ b/src/main/java/org/broadinstitute/dsm/db/ParticipantData.java @@ -50,10 +50,10 @@ public class ParticipantData { "ddp_participant_id = ?, ddp_instance_id = ?, field_type_id = ?, data = ?, last_changed = ?, changed_by = ? "; @ColumnName(DBConstants.PARTICIPANT_DATA_ID) - private String participantDataId; + private long participantDataId; @ColumnName(DBConstants.DDP_PARTICIPANT_ID) - private String ddpParticipantId; + private long ddpParticipantId; @ColumnName(DBConstants.FIELD_TYPE_ID) private String fieldTypeId; @@ -76,7 +76,7 @@ public ParticipantData() { } - public ParticipantData(String dataId, String fieldTypeId, String data) { + public ParticipantData(long dataId, String fieldTypeId, String data) { this.participantDataId = dataId; this.fieldTypeId = fieldTypeId; this.data = data; @@ -84,7 +84,7 @@ public ParticipantData(String dataId, String fieldTypeId, String data) { public static ParticipantData getParticipantDataObject(@NonNull ResultSet rs) throws SQLException { ParticipantData participantData = new ParticipantData( - rs.getString(DBConstants.PARTICIPANT_DATA_ID), + rs.getLong(DBConstants.PARTICIPANT_DATA_ID), rs.getString(DBConstants.FIELD_TYPE_ID), rs.getString(DBConstants.DATA) ); diff --git a/src/main/java/org/broadinstitute/dsm/db/Tissue.java b/src/main/java/org/broadinstitute/dsm/db/Tissue.java index 244275dd9..7646c7fa1 100644 --- a/src/main/java/org/broadinstitute/dsm/db/Tissue.java +++ b/src/main/java/org/broadinstitute/dsm/db/Tissue.java @@ -55,16 +55,16 @@ public class Tissue { primaryKey = DBConstants.TISSUE_ID, columnPrefix = "") @ColumnName(DBConstants.TISSUE_ID) - private String tissueId; + private long tissueId; @ColumnName(DBConstants.ONC_HISTORY_DETAIL_ID) - private String oncHistoryDetailId; + private long oncHistoryDetailId; @ColumnName (DBConstants.NOTES) private String notes; @ColumnName (DBConstants.COUNT_RECEIVED) - private Integer countReceived; + private long countReceived; @ColumnName (DBConstants.TISSUE_TYPE) private String tissueType; @@ -143,30 +143,30 @@ public Map getDynamicFields() { private String tissueSequence; @ColumnName (DBConstants.SCROLLS_COUNT) - private Integer scrollsCount; + private long scrollsCount; @ColumnName (DBConstants.USS_COUNT) - private Integer ussCount; + private long ussCount; @ColumnName (DBConstants.BLOCKS_COUNT) - private Integer blocksCount; + private long blocksCount; @ColumnName (DBConstants.H_E_COUNT) - private Integer hECount; + private long hECount; @JsonProperty("hECount") - public Integer gethECount() { + public long gethECount() { return hECount; } public Tissue() {} - public Tissue(String tissueId, String oncHistoryDetailId, String notes, Integer countReceived, String tissueType, + public Tissue(long tissueId, long oncHistoryDetailId, String notes, Integer countReceived, String tissueType, String tissueSite, String tumorType, String hE, String pathologyReport, String collaboratorSampleId, String blockSent, String scrollsReceived, String skId, String smId, String sentGp, String firstSmId, String additionalValuesJson, String expectedReturn, String returnDate, - String returnFedexId, String shlWorkNumber, String tumorPercentage, String tissueSequence, Integer scrollsCount, - Integer ussCount, Integer blocksCount, Integer hECount) { + String returnFedexId, String shlWorkNumber, String tumorPercentage, String tissueSequence, long scrollsCount, + long ussCount, long blocksCount, long hECount) { this.tissueId = tissueId; this.oncHistoryDetailId = oncHistoryDetailId; this.notes = notes; @@ -201,8 +201,8 @@ public static Tissue getTissue(@NonNull ResultSet rs) throws SQLException { if (StringUtils.isBlank(tissueId)) return null; Tissue tissue = new Tissue( - rs.getString(DBConstants.TISSUE_ID), - rs.getString(DBConstants.ONC_HISTORY_DETAIL_ID), + rs.getLong(DBConstants.TISSUE_ID), + rs.getLong(DBConstants.ONC_HISTORY_DETAIL_ID), rs.getString(DBConstants.DDP_TISSUE_ALIAS + DBConstants.ALIAS_DELIMITER + DBConstants.NOTES), rs.getInt(DBConstants.COUNT_RECEIVED), rs.getString(DBConstants.TISSUE_TYPE), @@ -224,10 +224,10 @@ public static Tissue getTissue(@NonNull ResultSet rs) throws SQLException { rs.getString(DBConstants.SHL_WORK_NUMBER), rs.getString(DBConstants.TUMOR_PERCENTAGE), rs.getString(DBConstants.TISSUE_SEQUENCE), - rs.getInt(DBConstants.SCROLLS_COUNT), - rs.getInt(DBConstants.USS_COUNT), - rs.getInt(DBConstants.BLOCKS_COUNT), - rs.getInt(DBConstants.H_E_COUNT)); + rs.getLong(DBConstants.SCROLLS_COUNT), + rs.getLong(DBConstants.USS_COUNT), + rs.getLong(DBConstants.BLOCKS_COUNT), + rs.getLong(DBConstants.H_E_COUNT)); return tissue; } diff --git a/src/main/java/org/broadinstitute/dsm/model/TissueList.java b/src/main/java/org/broadinstitute/dsm/model/TissueList.java index a3e5220d4..503977c2f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/TissueList.java +++ b/src/main/java/org/broadinstitute/dsm/model/TissueList.java @@ -64,7 +64,7 @@ public static List getAllTissueListsForRealm(String realm, String qu Tissue tissue = Tissue.getTissue(rs); TissueList tissueList = new TissueList(oncHistory, null, ptId); - if (!tissue.isDeleted() && StringUtils.isNotBlank(tissue.getTissueId())) { + if (!tissue.isDeleted()) { tissueList.setTissue(tissue); } results.add(tissueList); diff --git a/src/main/java/org/broadinstitute/dsm/model/at/SearchKitRequest.java b/src/main/java/org/broadinstitute/dsm/model/at/SearchKitRequest.java index 3b58c7bff..229daef7e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/at/SearchKitRequest.java +++ b/src/main/java/org/broadinstitute/dsm/model/at/SearchKitRequest.java @@ -41,7 +41,7 @@ public static ParticipantData findATKitRequest(@NonNull String mfBarcode) { try (PreparedStatement stmt = conn.prepareStatement(SQL_SELECT_KIT_REQUEST.concat(MF_BARCODE + " like \"%" + mfBarcode + "%\""))) { try (ResultSet rs = stmt.executeQuery()) { if (rs.next()) { - dbVals.resultValue = new ParticipantData(rs.getString(DBConstants.PARTICIPANT_DATA_ID), null, rs.getString(DBConstants.DATA)); + dbVals.resultValue = new ParticipantData(rs.getLong(DBConstants.PARTICIPANT_DATA_ID), null, rs.getString(DBConstants.DATA)); } } } diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java index 004ad6c3b..63eca1f27 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java @@ -261,9 +261,9 @@ private List collectData(DDPInstance ddpInstance) { private void mapTissueToProperOncHistoryDetail(List oncHistoryDetails, List tissues) { for (Tissue tissue : tissues) { - String oncHistoryDetailId = tissue.getOncHistoryDetailId(); + long oncHistoryDetailId = tissue.getOncHistoryDetailId(); oncHistoryDetails.stream() - .filter(oncHistoryDetail -> oncHistoryDetail.getOncHistoryDetailId().equals(oncHistoryDetailId)) + .filter(oncHistoryDetail -> oncHistoryDetail.getOncHistoryDetailId() == oncHistoryDetailId) .findFirst() .ifPresent(oncHistoryDetail -> oncHistoryDetail.getTissues().add(tissue)); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java index c0e58e2c9..571ffeb21 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java @@ -27,19 +27,16 @@ public void underscoresToCamelCase() { String fieldName3 = "column"; String fieldName4 = "COLUMN"; String fieldName5 = "columnName"; - String fieldName6 = "column.name"; String transformed = Util.underscoresToCamelCase(fieldName); String transformed2 = Util.underscoresToCamelCase(fieldName2); String transformed3 = Util.underscoresToCamelCase(fieldName3); String transformed4 = Util.underscoresToCamelCase(fieldName4); String transformed5 = Util.underscoresToCamelCase(fieldName5); - String transformed6 = Util.underscoresToCamelCase(fieldName6); assertEquals("columnName", transformed); assertEquals("columnName", transformed2); assertEquals("column", transformed3); assertEquals("column", transformed4); assertEquals("columnName", transformed5); - assertEquals("columnName", transformed6); } @@ -72,22 +69,8 @@ public void transformJsonToMap() { .build(); Map result = Util.transformObjectToMap(participantDataDto); - assertEquals("TEST", result.get("ddpInstance")); - assertEquals("VALUE", result.get("ddpValue")); - } - - @Test - public void convertToMap() { - String fieldName = "tissue"; - List fieldValue = List.of(new Tissue("11", "22", - null, null, null, null, null, null, null, null, null, null, - null, null, null, null, null, null, null, null, null, null, - null, null, null, null, null)); - - Map stringObjectMap = Util.convertToMap(fieldName, fieldValue); - - System.out.println(); - System.out.println(); + assertEquals("TEST", ((Map) result.get("dynamicFields")).get("ddpInstance")); + assertEquals("VALUE", ((Map) result.get("dynamicFields")).get("ddpValue")); } @Test @@ -117,30 +100,17 @@ class MockClass { @Test public void camelCaseToPascalSnakeCase() { - String camelCase = "registrationType"; - String camelCaseVal = "test"; - - String camelCase2 = "medicalRecordsReleaseObtained"; - String camelCase2Val = "NO"; - - String pascalSnakeCase = Util.camelCaseToPascalSnakeCase(camelCase); - String pascalSnakeCaseVal = Util.camelCaseToPascalSnakeCase(camelCaseVal); + String camelCase1 = "registrationType"; + String camelCase2 = "test"; + String camelCase3 = "medicalRecordsReleaseObtained"; + String pascalSnakeCase1 = Util.camelCaseToPascalSnakeCase(camelCase1); String pascalSnakeCase2 = Util.camelCaseToPascalSnakeCase(camelCase2); - String pascalSnakeCase2Val = Util.camelCaseToPascalSnakeCase(camelCase2Val); - - assertEquals("REGISTRATION_TYPE", pascalSnakeCase); - assertEquals("TEST", pascalSnakeCaseVal); + String pascalSnakeCase3 = Util.camelCaseToPascalSnakeCase(camelCase3); - assertEquals("MEDICAL_RECORDS_RELEASE_OBTAINED", pascalSnakeCase2); - assertEquals("NO", pascalSnakeCase2Val); - } - - @Test - public void camelCaseIfContainsDot() { - String field = "dynamicFields.Scooby"; - String converted = Util.underscoresToCamelCase(field); - assertEquals("dynamicFields.scooby", converted); + assertEquals("REGISTRATION_TYPE", pascalSnakeCase1); + assertEquals("TEST", pascalSnakeCase2); + assertEquals("MEDICAL_RECORDS_RELEASE_OBTAINED", pascalSnakeCase3); } } \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigratorTest.java index f1c0e2ef5..2fe97beb8 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigratorTest.java @@ -28,15 +28,15 @@ public void transformObject() { } private List mockData() { - List fieldValue = new ArrayList<>(List.of(new Tissue("11", "22", + List fieldValue = new ArrayList<>(List.of(new Tissue(11, 22, null, null, null, "awdwadawdawdawd", null, null, null, null, null, null, null, null, "Awdawd", null, null, null, null, null, null, null, - null, null, null, null, null), new Tissue("555", "777", + null, 0, 0, 0, 0), new Tissue(555, 777, null, null, null, null, null, null, null, "awdawd", null, null, null, null, "awdawddwa", null, null, null, null, null, null, null, - null, null, null, null, null))); + null, 0, 0, 0, 0))); OncHistoryDetail oncHistoryDetail = - new OncHistoryDetail("23", null, null, null, null, null, null, null, null, null, null, null, null, null, null, + new OncHistoryDetail(23, 0, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, fieldValue, null, null, false); return Collections.singletonList(oncHistoryDetail); } From dc260899ec4840292ce68c6ce6435f1a62d79321 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 3 Jan 2022 17:05:43 +0400 Subject: [PATCH 281/405] DDP-7121 | change string to longs --- .../org/broadinstitute/dsm/db/KitDiscard.java | 4 +- .../dsm/db/KitRequestCreateLabel.java | 2 +- .../dsm/db/KitRequestShipping.java | 46 ++++++++++--------- .../broadinstitute/dsm/db/MedicalRecord.java | 10 ++-- .../org/broadinstitute/dsm/db/OncHistory.java | 6 +-- .../broadinstitute/dsm/db/Participant.java | 23 +++++----- .../broadinstitute/dsm/model/KitRequest.java | 7 +-- .../dsm/model/KitRequestExternal.java | 3 +- .../dsm/model/KitShippingIds.java | 4 +- .../dsm/model/KitUploadObject.java | 2 +- .../dsm/model/at/ReceiveKitRequest.java | 2 +- .../dsm/route/KitDeactivationRoute.java | 3 +- .../dsm/route/KitExpressRoute.java | 3 +- .../org/broadinstitute/dsm/util/KitUtil.java | 13 ++++-- .../org/broadinstitute/dsm/util/PDFAudit.java | 2 +- .../util/externalShipper/GBFRequestUtil.java | 11 +++-- .../dsm/DDPRequestRouteTest.java | 6 +-- .../dsm/model/elastic/UtilTest.java | 4 +- .../export/process/SingleProcessorTest.java | 4 +- .../DynamicFieldsMappingMigratorTest.java | 4 +- 20 files changed, 87 insertions(+), 72 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/KitDiscard.java b/src/main/java/org/broadinstitute/dsm/db/KitDiscard.java index 7464c3c6f..7892f7e94 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitDiscard.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitDiscard.java @@ -261,12 +261,12 @@ else if (IMAGE_FILE.equals(pathName)) { } } - public static String addKitToDiscard(@NonNull String kitRequestId, @NonNull String action) { + public static String addKitToDiscard(@NonNull long kitRequestId, @NonNull String action) { SimpleResult results = inTransaction((conn) -> { SimpleResult dbVals = new SimpleResult(); try (PreparedStatement stmt = conn.prepareStatement(TransactionWrapper.getSqlFromConfig(ApplicationConfigConstants.INSERT_KIT_DISCARD), Statement.RETURN_GENERATED_KEYS)) { stmt.setString(1, action); - stmt.setString(2, kitRequestId); + stmt.setLong(2, kitRequestId); int result = stmt.executeUpdate(); if (result == 1) { try (ResultSet rs = stmt.getGeneratedKeys()) { diff --git a/src/main/java/org/broadinstitute/dsm/db/KitRequestCreateLabel.java b/src/main/java/org/broadinstitute/dsm/db/KitRequestCreateLabel.java index 792f35766..cd981e7a3 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitRequestCreateLabel.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitRequestCreateLabel.java @@ -112,7 +112,7 @@ public static void updateKitLabelRequested(String realm, String type, @NonNull S if (!kitsNoLabel.isEmpty()) { for (KitRequestCreateLabel uploadedKit : kitsNoLabel) { - KitRequestShipping.updateKit(uploadedKit.getDsmKitId(), userId); + KitRequestShipping.updateKit(Long.parseLong(uploadedKit.getDsmKitId()), userId); } } logger.info("Done triggering label creation"); diff --git a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java index 5a606d911..308257664 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java @@ -145,7 +145,7 @@ public class KitRequestShipping extends KitRequest { private static final String SEARCH_MF_BAR = "MF_BAR"; @ColumnName(DBConstants.DSM_KIT_ID) - private String dsmKitId; + private long dsmKitId; @ColumnName(DBConstants.LABEL_URL_TO) private String labelUrlTo; @@ -236,22 +236,24 @@ public class KitRequestShipping extends KitRequest { public KitRequestShipping() {} - public KitRequestShipping(String collaboratorParticipantId, String kitTypeName, String dsmKitRequestId, long scanDate, boolean error, long receiveDate, long deactivatedDate, String testResult, + public KitRequestShipping(String collaboratorParticipantId, String kitTypeName, long dsmKitRequestId, long scanDate, boolean error, + long receiveDate, long deactivatedDate, String testResult, String upsTrackingStatus, String upsReturnStatus, String externalOrderStatus, String externalOrderNumber, long externalOrderDate, boolean careEvolve, String uploadReason) { - this(null, collaboratorParticipantId, null, null, null, kitTypeName, dsmKitRequestId, null, null, null, + this(null, collaboratorParticipantId, null, null, null, kitTypeName, dsmKitRequestId, 0, null, null, null, null, null, null, scanDate, error, null, receiveDate, null, deactivatedDate, null, null, false, null, 0, null, externalOrderNumber, false, externalOrderStatus, null, testResult, upsTrackingStatus, upsReturnStatus, externalOrderDate, false, uploadReason, null, null, null); } public KitRequestShipping(String participantId, String collaboratorParticipantId, String dsmKitId, String realm, String trackingNumberTo, String receiveDateString, String hruid, String gender) { - this(participantId, collaboratorParticipantId, null, null, realm, null, null, null, null, null, + this(participantId, collaboratorParticipantId, null, null, realm, null, 0, 0, null, null, trackingNumberTo, null, null, null, 0, false, null, 0, null, 0, null, dsmKitId, false, null, 0, null, null, false, null, null, null, null, null, 0, false, null, receiveDateString, hruid, gender); } - public KitRequestShipping(String dsmKitRequestId, String dsmKitId, String easypostToId, String easypostAddressId, boolean error, String message) { + public KitRequestShipping(long dsmKitRequestId, long dsmKitId, String easypostToId, String easypostAddressId, boolean error, + String message) { this(null, null, null, null, null, null, dsmKitRequestId, dsmKitId, null, null, null, null, null, null, 0, error, message, 0, easypostAddressId, 0, null, null, false, easypostToId, 0, null, null, false, null, null, null, null, null, 0, false, null, @@ -260,7 +262,7 @@ public KitRequestShipping(String dsmKitRequestId, String dsmKitId, String easypo // shippingId = ddp_label !!! public KitRequestShipping(String participantId, String collaboratorParticipantId, String bspCollaboratorSampleId, String shippingId, String realm, - String kitTypeName, String dsmKitRequestId, String dsmKitId, String labelUrlTo, String labelUrlReturn, + String kitTypeName, long dsmKitRequestId, long dsmKitId, String labelUrlTo, String labelUrlReturn, String trackingNumberTo, String trackingNumberReturn, String trackingUrlTo, String trackingUrlReturn, long scanDate, boolean error, String message, long receiveDate, String easypostAddressId, long deactivatedDate, String deactivationReason, @@ -316,8 +318,8 @@ public static KitRequestShipping getKitRequestShipping(@NonNull ResultSet rs) th rs.getString(DBConstants.DSM_LABEL), rs.getString(DBConstants.INSTANCE_NAME), rs.getString(DBConstants.KIT_TYPE_NAME), - rs.getString(DBConstants.DSM_KIT_REQUEST_ID), - rs.getString(DBConstants.DSM_KIT_ID), + rs.getLong(DBConstants.DSM_KIT_REQUEST_ID), + rs.getLong(DBConstants.DSM_KIT_ID), rs.getString(DBConstants.DSM_LABEL_TO), rs.getString(DBConstants.DSM_LABEL_RETURN), rs.getString(DBConstants.DSM_TRACKING_TO), @@ -441,10 +443,11 @@ private static void addUPSKitRequest(ResultSet rs, Map k.getDsmKitRequestId().equals(krs.getDsmKitRequestId())).findFirst().isPresent(); + boolean kitPresent = kitRequestList.stream().filter(k -> k.getDsmKitRequestId() == krs.getDsmKitRequestId()).findFirst().isPresent(); if (kitRequestList != null && kitPresent) { - KitRequestShipping kit = kitRequestList.stream().filter(k -> k.getDsmKitRequestId().equals(krs.getDsmKitRequestId())).findFirst().get(); - kitRequestList.removeIf((k -> k.getDsmKitRequestId().equals(krs.getDsmKitRequestId()))); + KitRequestShipping kit = + kitRequestList.stream().filter(k -> k.getDsmKitRequestId() == krs.getDsmKitRequestId()).findFirst().get(); + kitRequestList.removeIf((k -> k.getDsmKitRequestId() == krs.getDsmKitRequestId())); if (StringUtils.isBlank(kit.getUpsTrackingStatus()) && StringUtils.isNotBlank(krs.getUpsTrackingStatus())) { kit.setUpsTrackingStatus(krs.getUpsTrackingStatus()); } @@ -494,7 +497,7 @@ public static List getKitRequestsByParticipant(@NonNull Stri KitRequestShipping kitRequest = new KitRequestShipping( rs.getString(DBConstants.COLLABORATOR_PARTICIPANT_ID), rs.getString(DBConstants.KIT_TYPE_NAME), - rs.getString(DBConstants.DSM_KIT_REQUEST_ID), + rs.getLong(DBConstants.DSM_KIT_REQUEST_ID), rs.getLong(DBConstants.DSM_SCAN_DATE), false, rs.getLong(DBConstants.DSM_RECEIVE_DATE), @@ -727,14 +730,15 @@ private static void getAddressPerEasypost(DDPInstance ddpInstance, KitRequestShi } } - public static void deactivateKitRequest(@NonNull String kitRequestId, @NonNull String reason, String easypostApiKey, @NonNull String userId) { + public static void deactivateKitRequest(@NonNull long kitRequestId, @NonNull String reason, String easypostApiKey, + @NonNull String userId) { SimpleResult results = inTransaction((conn) -> { SimpleResult dbVals = new SimpleResult(); try (PreparedStatement stmt = conn.prepareStatement(UPDATE_KIT_DEACTIVATION)) { stmt.setLong(1, System.currentTimeMillis()); stmt.setString(2, reason); stmt.setString(3, userId); - stmt.setString(4, kitRequestId); + stmt.setLong(4, kitRequestId); int result = stmt.executeUpdate(); if (result == 1) { logger.info("Deactivated kitRequest w/ dsm_kit_request_id " + kitRequestId); @@ -909,13 +913,13 @@ public static void writeNewKit(String kitRequestId, String addressIdTo, String e } // update kit with label trigger user and date - public static void updateKit(String dsmKitId, String userId) { + public static void updateKit(long dsmKitId, String userId) { SimpleResult results = inTransaction((conn) -> { SimpleResult dbVals = new SimpleResult(); try (PreparedStatement stmt = conn.prepareStatement(SQL_UPDATE_KIT)) { stmt.setString(1, userId); stmt.setLong(2, System.currentTimeMillis()); - stmt.setString(3, dsmKitId); + stmt.setLong(3, dsmKitId); int result = stmt.executeUpdate(); if (result > 1) { @@ -1251,7 +1255,7 @@ private static Shipment getEasyPostShipment(@NonNull EasyPostUtil easyPostUtil, return null; } - public static void refundKit(@NonNull String kitRequestId, @NonNull String easypostApiKey) { + public static void refundKit(@NonNull long kitRequestId, @NonNull String easypostApiKey) { KitShippingIds shippingIds = KitShippingIds.getKitShippingIds(kitRequestId, easypostApiKey); if (shippingIds != null) { String message = ""; @@ -1281,13 +1285,13 @@ public static void refundKit(@NonNull String kitRequestId, @NonNull String easyp } } - public static void updateKitError(@NonNull String kitRequestId, @NonNull String message) { + public static void updateKitError(@NonNull long kitRequestId, @NonNull String message) { SimpleResult results = inTransaction((conn) -> { SimpleResult dbVals = new SimpleResult(); try (PreparedStatement stmt = conn.prepareStatement(TransactionWrapper.getSqlFromConfig(ApplicationConfigConstants.UPDATE_KIT_ERROR))) { stmt.setInt(1, 1); stmt.setString(2, message); - stmt.setString(3, kitRequestId); + stmt.setLong(3, kitRequestId); int result = stmt.executeUpdate(); if (result == 1) { logger.info("Updated error/message for kit request w/ dsm_kit_request_id " + kitRequestId); @@ -1326,8 +1330,8 @@ public static void reactivateKitRequest(@NonNull String kitRequestId, String mes } if (rs.next()) { dbVals.resultValue = new KitRequestShipping( - rs.getString(DBConstants.DSM_KIT_REQUEST_ID), - rs.getString(DBConstants.DSM_KIT_ID), + rs.getLong(DBConstants.DSM_KIT_REQUEST_ID), + rs.getLong(DBConstants.DSM_KIT_ID), rs.getString(DBConstants.EASYPOST_TO_ID), rs.getString(DBConstants.EASYPOST_ADDRESS_ID_TO), false, null diff --git a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java index c820ad6fa..20bab003d 100644 --- a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java +++ b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java @@ -85,7 +85,7 @@ public class MedicalRecord { primaryKey = DBConstants.DDP_INSTITUTION_ID, columnPrefix = "") @ColumnName (DBConstants.DDP_INSTITUTION_ID) - private long ddpInstitutionId; + private String ddpInstitutionId; @TableName ( name = DBConstants.DDP_PARTICIPANT, @@ -363,7 +363,7 @@ public Map getDynamicFields() { @ColumnName (DBConstants.PATHOLOGY_PRESENT) private String pathologyPresent; - public MedicalRecord(long medicalRecordId, long institutionId, long ddpInstitutionId, String type) { + public MedicalRecord(long medicalRecordId, long institutionId, String ddpInstitutionId, String type) { this.medicalRecordId = medicalRecordId; this.institutionId = institutionId; this.ddpInstitutionId = ddpInstitutionId; @@ -372,7 +372,7 @@ public MedicalRecord(long medicalRecordId, long institutionId, long ddpInstituti public MedicalRecord() {} - public MedicalRecord(long medicalRecordId, long institutionId, long ddpInstitutionId, String type, + public MedicalRecord(long medicalRecordId, long institutionId, String ddpInstitutionId, String type, String name, String contact, String phone, String fax, String faxSent, String faxSentBy, String faxConfirmed, String faxSent2, String faxSent2By, String faxConfirmed2, @@ -423,7 +423,7 @@ public static MedicalRecord getMedicalRecord(@NonNull ResultSet rs) throws SQLEx MedicalRecord medicalRecord = new MedicalRecord( rs.getLong(DBConstants.MEDICAL_RECORD_ID), rs.getLong(DBConstants.INSTITUTION_ID), - rs.getLong(DBConstants.DDP_INSTITUTION_ID), + rs.getString(DBConstants.DDP_INSTITUTION_ID), rs.getString(DBConstants.TYPE), rs.getString(DBConstants.NAME), rs.getString(DBConstants.CONTACT), @@ -552,7 +552,7 @@ public static List getInstitutions(@NonNull String realm, @NonNul medicalRecords.add(new MedicalRecord( rs.getLong(DBConstants.MEDICAL_RECORD_ID), rs.getLong(DBConstants.INSTITUTION_ID), - rs.getLong(DBConstants.DDP_INSTITUTION_ID), + rs.getString(DBConstants.DDP_INSTITUTION_ID), rs.getString(DBConstants.TYPE))); } } diff --git a/src/main/java/org/broadinstitute/dsm/db/OncHistory.java b/src/main/java/org/broadinstitute/dsm/db/OncHistory.java index c1956512b..0d7936f3c 100644 --- a/src/main/java/org/broadinstitute/dsm/db/OncHistory.java +++ b/src/main/java/org/broadinstitute/dsm/db/OncHistory.java @@ -20,21 +20,21 @@ public class OncHistory { private static final String SQL_UPDATE_ONC_HISTORY = "UPDATE ddp_onc_history SET created = ?, last_changed = ?, changed_by = ? WHERE participant_id = ? AND created IS NULL"; - private String participantId; + private long participantId; private String created; private String reviewed; private String changedBy; public OncHistory() {} - public OncHistory (String participantId, String created, String reviewed, String changedBy) { + public OncHistory (long participantId, String created, String reviewed, String changedBy) { this.participantId = participantId; this.created = created; this.reviewed = reviewed; this.changedBy = changedBy; } - public String getParticipantId() { + public long getParticipantId() { return participantId; } diff --git a/src/main/java/org/broadinstitute/dsm/db/Participant.java b/src/main/java/org/broadinstitute/dsm/db/Participant.java index 9d3b1f9d6..35371ae44 100644 --- a/src/main/java/org/broadinstitute/dsm/db/Participant.java +++ b/src/main/java/org/broadinstitute/dsm/db/Participant.java @@ -43,7 +43,7 @@ public class Participant { "WHERE realm.instance_name = ? "; @ColumnName(DBConstants.PARTICIPANT_ID) - private String participantId; + private long participantId; @ColumnName(DBConstants.DDP_PARTICIPANT_ID) private String ddpParticipantId; @@ -54,7 +54,7 @@ public class Participant { primaryKey = DBConstants.PARTICIPANT_ID, columnPrefix = "") @ColumnName (DBConstants.ASSIGNEE_ID_MR) - private String assigneeIdMr; + private long assigneeIdMr; @TableName ( name = DBConstants.DDP_PARTICIPANT, @@ -62,7 +62,7 @@ public class Participant { primaryKey = DBConstants.PARTICIPANT_ID, columnPrefix = "") @ColumnName (DBConstants.ASSIGNEE_ID_TISSUE) - private String assigneeIdTissue; + private long assigneeIdTissue; private String realm; @TableName ( @@ -87,7 +87,7 @@ public class Participant { primaryKey = DBConstants.PARTICIPANT_ID, columnPrefix = "") @ColumnName (DBConstants.CR_SENT) - private String paperCRSent; + private String crSent; @TableName ( name = DBConstants.DDP_PARTICIPANT_RECORD, @@ -95,7 +95,7 @@ public class Participant { primaryKey = DBConstants.PARTICIPANT_ID, columnPrefix = "") @ColumnName (DBConstants.CR_RECEIVED) - private String paperCRReceived; + private String crReceived; @TableName ( name = DBConstants.DDP_PARTICIPANT_RECORD, @@ -150,8 +150,8 @@ public Map getDynamicFields() { public Participant() {} - public Participant(String participantId, String ddpParticipantId, String assigneeIdMr, String assigneeIdTissue, String instanceName, - String createdOncHistory, String reviewedOncHistory, String paperCRSent, String paperCRReceived, String notes, + public Participant(long participantId, String ddpParticipantId, long assigneeIdMr, long assigneeIdTissue, String instanceName, + String createdOncHistory, String reviewedOncHistory, String crSent, String crReceived, String notes, boolean minimalMr, boolean abstractionReady, String additionalValuesJson, long exitDate) { this.participantId = participantId; this.ddpParticipantId = ddpParticipantId; @@ -160,8 +160,8 @@ public Participant(String participantId, String ddpParticipantId, String assigne this.realm = instanceName; this.createdOncHistory = createdOncHistory; this.reviewedOncHistory = reviewedOncHistory; - this.paperCRSent = paperCRSent; - this.paperCRReceived = paperCRReceived; + this.crSent = crSent; + this.crReceived = crReceived; this.notes = notes; this.minimalMr = minimalMr; this.abstractionReady = abstractionReady; @@ -182,9 +182,10 @@ public static Participant getParticipant(@NonNull Map assignee assigneeTissue = assignees.get(assigneeIdTissue).getName(); } } - Participant participant = new Participant(rs.getString(DBConstants.PARTICIPANT_ID), + Participant participant = new Participant(rs.getLong(DBConstants.PARTICIPANT_ID), rs.getString(DBConstants.DDP_PARTICIPANT_ID), - assigneeMR, assigneeTissue, realm, + rs.getLong(DBConstants.ASSIGNEE_ID_MR), + rs.getLong(DBConstants.ASSIGNEE_ID_TISSUE), realm, rs.getString(DBConstants.ONC_HISTORY_CREATED), rs.getString(DBConstants.ONC_HISTORY_REVIEWED), rs.getString(DBConstants.CR_SENT), diff --git a/src/main/java/org/broadinstitute/dsm/model/KitRequest.java b/src/main/java/org/broadinstitute/dsm/model/KitRequest.java index 2314c4bb2..a2af4c89f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/KitRequest.java +++ b/src/main/java/org/broadinstitute/dsm/model/KitRequest.java @@ -17,7 +17,7 @@ public class KitRequest { @ColumnName(DBConstants.DSM_KIT_REQUEST_ID) - private String dsmKitRequestId; + private long dsmKitRequestId; private String participantId; private String shortId; private String shippingId; @@ -37,10 +37,11 @@ public class KitRequest { public KitRequest() {} public KitRequest(String participantId, String shortId, DDPParticipant participant, String externalOrderNumber) { - this(null, participantId, shortId, null, externalOrderNumber, participant, null, null, null); + this(0, participantId, shortId, null, externalOrderNumber, participant, null, null, null); } - public KitRequest(String dsmKitRequestId, String participantId, String shortId, String shippingId, String externalOrderNumber, DDPParticipant participant, String externalOrderStatus, String externalKitName, Long externalOrderDate) { + public KitRequest(long dsmKitRequestId, String participantId, String shortId, String shippingId, String externalOrderNumber, + DDPParticipant participant, String externalOrderStatus, String externalKitName, Long externalOrderDate) { this.dsmKitRequestId = dsmKitRequestId; this.participantId = participantId; this.shortId = shortId; diff --git a/src/main/java/org/broadinstitute/dsm/model/KitRequestExternal.java b/src/main/java/org/broadinstitute/dsm/model/KitRequestExternal.java index c3fe15df3..0f3814066 100644 --- a/src/main/java/org/broadinstitute/dsm/model/KitRequestExternal.java +++ b/src/main/java/org/broadinstitute/dsm/model/KitRequestExternal.java @@ -23,7 +23,8 @@ public class KitRequestExternal extends KitRequest { private static final String SQL_UPDATE_KIT_REQUEST_EXTERNAL_SHIPPER_STATUS = "UPDATE ddp_kit_request SET external_order_status = ?, external_order_date = ? WHERE dsm_kit_request_id = ? AND NOT external_order_status <=> ?"; private static final String SQL_UPDATE_KIT_REQUEST_EXTERNAL_SHIPPER_RESPONSE = "UPDATE ddp_kit_request SET external_response = ? WHERE dsm_kit_request_id = ?"; - public KitRequestExternal(String dsmKitRequestId, String participantId, String shortId, String shippingId, String externalOrderNumber, DDPParticipant participant, + public KitRequestExternal(long dsmKitRequestId, String participantId, String shortId, String shippingId, String externalOrderNumber, + DDPParticipant participant, String externalOrderStatus, String externalKitName) { super(dsmKitRequestId, participantId, shortId, shippingId, externalOrderNumber, participant, externalOrderStatus, externalKitName, null); } diff --git a/src/main/java/org/broadinstitute/dsm/model/KitShippingIds.java b/src/main/java/org/broadinstitute/dsm/model/KitShippingIds.java index bf6881281..c0811973c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/KitShippingIds.java +++ b/src/main/java/org/broadinstitute/dsm/model/KitShippingIds.java @@ -25,11 +25,11 @@ public KitShippingIds(@NonNull String easyPostApiKey, String easyPostShipmentToI this.easyPostShipmentReturnId = easyPostShipmentReturnId; } - public static KitShippingIds getKitShippingIds(@NonNull String kitRequestId, @NonNull String easypostApiKey) { + public static KitShippingIds getKitShippingIds(@NonNull long kitRequestId, @NonNull String easypostApiKey) { SimpleResult results = inTransaction((conn) -> { SimpleResult dbVals = new SimpleResult(); try (PreparedStatement stmt = conn.prepareStatement(KitRequestShipping.SQL_SELECT_KIT)) { - stmt.setString(1, kitRequestId); + stmt.setLong(1, kitRequestId); try (ResultSet rs = stmt.executeQuery()) { int numRows = 0; while (rs.next()) { diff --git a/src/main/java/org/broadinstitute/dsm/model/KitUploadObject.java b/src/main/java/org/broadinstitute/dsm/model/KitUploadObject.java index bf1b7bae0..4f04b3c2f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/KitUploadObject.java +++ b/src/main/java/org/broadinstitute/dsm/model/KitUploadObject.java @@ -17,7 +17,7 @@ public class KitUploadObject extends KitRequest { public KitUploadObject(String externalOrderNumber, String ddpParticipantId, String shortId, String firstName, String lastName, String street1, String street2, String city, String state, String postalCode, String country, String phoneNumber) { - super(null, ddpParticipantId, shortId, null, externalOrderNumber, null, null, null, null); + super(0, ddpParticipantId, shortId, null, externalOrderNumber, null, null, null, null); this.firstName = firstName; this.lastName = lastName; this.street1 = street1; diff --git a/src/main/java/org/broadinstitute/dsm/model/at/ReceiveKitRequest.java b/src/main/java/org/broadinstitute/dsm/model/at/ReceiveKitRequest.java index 42581da4f..3a701fd83 100644 --- a/src/main/java/org/broadinstitute/dsm/model/at/ReceiveKitRequest.java +++ b/src/main/java/org/broadinstitute/dsm/model/at/ReceiveKitRequest.java @@ -58,7 +58,7 @@ public static boolean receiveATKitRequest(@NonNull NotificationUtil notification return false; } - private static boolean updateData(@NonNull String data, @NonNull String participantDataId) { + private static boolean updateData(@NonNull String data, @NonNull long participantDataId) { SimpleResult results = inTransaction((conn) -> { SimpleResult dbVals = new SimpleResult(); try (PreparedStatement stmt = conn.prepareStatement(SQL_UPDATE_KIT_REQUEST)){ diff --git a/src/main/java/org/broadinstitute/dsm/route/KitDeactivationRoute.java b/src/main/java/org/broadinstitute/dsm/route/KitDeactivationRoute.java index 93cd80231..b3092203c 100644 --- a/src/main/java/org/broadinstitute/dsm/route/KitDeactivationRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/KitDeactivationRoute.java @@ -48,7 +48,8 @@ public Object processRequest(Request request, Response response, String userId) if (deactivate) { JsonObject jsonObject = new JsonParser().parse(request.body()).getAsJsonObject(); String reason = jsonObject.get("reason").getAsString(); - KitRequestShipping.deactivateKitRequest(kitRequestId, reason, DSMServer.getDDPEasypostApiKey(realm), userIdRequest); + KitRequestShipping.deactivateKitRequest(Long.parseLong(kitRequestId), reason, + DSMServer.getDDPEasypostApiKey(realm), userIdRequest); } else { QueryParamsMap queryParams = request.queryMap(); diff --git a/src/main/java/org/broadinstitute/dsm/route/KitExpressRoute.java b/src/main/java/org/broadinstitute/dsm/route/KitExpressRoute.java index b2f33ecea..ccb8dae17 100644 --- a/src/main/java/org/broadinstitute/dsm/route/KitExpressRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/KitExpressRoute.java @@ -71,7 +71,8 @@ public Object processRequest(Request request, Response response, String userId) public void expressKitRequest(@NonNull String kitRequestId, @NonNull String userId) { KitRequestShipping kitRequest = KitRequestShipping.getKitRequest(kitRequestId); //deactivate kit which is already in db and refund the label - KitRequestShipping.deactivateKitRequest(kitRequestId, KitRequestShipping.DEACTIVATION_REASON, DSMServer.getDDPEasypostApiKey(kitRequest.getRealm()), userId); + KitRequestShipping.deactivateKitRequest(Long.valueOf(kitRequestId), KitRequestShipping.DEACTIVATION_REASON, + DSMServer.getDDPEasypostApiKey(kitRequest.getRealm()), userId); //add new kit into db KitRequestShipping.reactivateKitRequest(kitRequestId); diff --git a/src/main/java/org/broadinstitute/dsm/util/KitUtil.java b/src/main/java/org/broadinstitute/dsm/util/KitUtil.java index 83c052e86..50faf95da 100644 --- a/src/main/java/org/broadinstitute/dsm/util/KitUtil.java +++ b/src/main/java/org/broadinstitute/dsm/util/KitUtil.java @@ -141,7 +141,9 @@ public static void createLabel(List kitsLabelTriggered) { KitRequestShipping.updateRequest(kitLabelTriggered, ddpParticipant, kitLabelTriggered.getKitTyp(), kitLabelTriggered.getKitRequestSettings()); } else { - KitRequestShipping.deactivateKitRequest(kitLabelTriggered.getDsmKitRequestId(), "Participant not found", null, "System"); + KitRequestShipping.deactivateKitRequest(Long.parseLong(kitLabelTriggered.getDsmKitRequestId()), "Participant not found", + null, + "System"); logger.error("Didn't find participant " + kitLabelTriggered.getDdpParticipantId()); } } @@ -447,8 +449,8 @@ private static List getKitRequestsToCheckStatus(@NonNull Str try (ResultSet rs = stmt.executeQuery()) { while (rs.next()) { KitRequestShipping kitRequest = new KitRequestShipping( - rs.getString(DBConstants.DSM_KIT_REQUEST_ID), - rs.getString(DBConstants.DSM_KIT_ID), + rs.getLong(DBConstants.DSM_KIT_REQUEST_ID), + rs.getLong(DBConstants.DSM_KIT_ID), rs.getString(DBConstants.EASYPOST_TO_ID), rs.getString(DBConstants.EASYPOST_ADDRESS_ID_TO), rs.getBoolean(DBConstants.ERROR), @@ -471,13 +473,14 @@ private static List getKitRequestsToCheckStatus(@NonNull Str } // update kit with label trigger user and date - private static SimpleResult updateKit(@NonNull Connection conn, String status, Long date, String message, String dsmKitId) { + private static SimpleResult updateKit(@NonNull Connection conn, String status, Long date, String message, + long dsmKitId) { SimpleResult dbVals = new SimpleResult(); try (PreparedStatement stmt = conn.prepareStatement(SQL_UPDATE_KIT)) { stmt.setString(1, status); stmt.setLong(2, date); stmt.setString(3, message); - stmt.setString(4, dsmKitId); + stmt.setLong(4, dsmKitId); int result = stmt.executeUpdate(); if (result != 1) { diff --git a/src/main/java/org/broadinstitute/dsm/util/PDFAudit.java b/src/main/java/org/broadinstitute/dsm/util/PDFAudit.java index d59bc0aa9..70ad828e1 100644 --- a/src/main/java/org/broadinstitute/dsm/util/PDFAudit.java +++ b/src/main/java/org/broadinstitute/dsm/util/PDFAudit.java @@ -36,7 +36,7 @@ public void checkAndSavePDF() { DDPRequestUtil.makeNonStandardPDF(instance, request.getParticipantId(), request.getCreatedBy(), request.getKitTypeName()); } } - newAuditKit = Math.max(newAuditKit, Integer.parseInt(request.getDsmKitRequestId())); + newAuditKit = Math.max(newAuditKit, request.getDsmKitRequestId()); } } catch (Exception e) { diff --git a/src/main/java/org/broadinstitute/dsm/util/externalShipper/GBFRequestUtil.java b/src/main/java/org/broadinstitute/dsm/util/externalShipper/GBFRequestUtil.java index c95a55047..3a3caf9a3 100644 --- a/src/main/java/org/broadinstitute/dsm/util/externalShipper/GBFRequestUtil.java +++ b/src/main/java/org/broadinstitute/dsm/util/externalShipper/GBFRequestUtil.java @@ -253,7 +253,10 @@ else if (status.getOrderStatus().contains(SHIPPED) && (StringUtils.isBlank(kit.g if (kitDDPNotification != null) { logger.info("Triggering DDP for shipped kit with external order number: " + kit.getExternalOrderNumber()); if (gbfShippedTriggerDSSDelivered) { - KitDDPNotification kitDeliveredNotification = KitDDPNotification.getKitDDPNotification(conn, SQL_SELECT_KIT_FOR_NOTIFICATION_EXTERNAL_SHIPPER + SELECT_BY_EXTERNAL_ORDER_NUMBER, new String[] { DELIVERED, String.valueOf(instanceId), kit.getDsmKitRequestId(), kit.getExternalOrderNumber() }, 1); + KitDDPNotification kitDeliveredNotification = KitDDPNotification.getKitDDPNotification(conn, + SQL_SELECT_KIT_FOR_NOTIFICATION_EXTERNAL_SHIPPER + SELECT_BY_EXTERNAL_ORDER_NUMBER, + new String[] { DELIVERED, String.valueOf(instanceId), String.valueOf(kit.getDsmKitRequestId()), + kit.getExternalOrderNumber() }, 1); if (kitDeliveredNotification != null) { logger.info("Triggering DDP for kit 'DELIVERED' with external order number: " + kit.getExternalOrderNumber()); EventUtil.triggerDDP(conn, kitDeliveredNotification); @@ -389,7 +392,7 @@ private void updateOrderStatusForPendingKitRequests(int instanceId, String query while (rs.next()) { String ddpParticipantId = rs.getString(DBConstants.DDP_PARTICIPANT_ID); if (StringUtils.isNotBlank(ddpParticipantId)) { - KitRequest kitRequest = new KitRequest(rs.getString(DBConstants.DSM_KIT_REQUEST_ID), ddpParticipantId, + KitRequest kitRequest = new KitRequest(rs.getLong(DBConstants.DSM_KIT_REQUEST_ID), ddpParticipantId, null, null, rs.getString(DBConstants.EXTERNAL_ORDER_NUMBER), null, rs.getString(DBConstants.EXTERNAL_ORDER_STATUS), rs.getString("subkits." + DBConstants.EXTERNAL_KIT_NAME), @@ -447,7 +450,7 @@ public ArrayList getKitRequestsNotDone(int instanceId, String que // DDPParticipant ddpParticipant = ElasticSearchUtil.getParticipantAsDDPParticipant(participantsESData, ddpParticipantId); // logger.info("ddpParticipant found: "+ddpParticipant.getParticipantId()); // if (ddpParticipant != null) { - kitRequests.add(new KitRequest(rs.getString(DBConstants.DSM_KIT_REQUEST_ID), ddpParticipantId, + kitRequests.add(new KitRequest(rs.getLong(DBConstants.DSM_KIT_REQUEST_ID), ddpParticipantId, null, null, rs.getString(DBConstants.EXTERNAL_ORDER_NUMBER), null, rs.getString(DBConstants.EXTERNAL_ORDER_STATUS), rs.getString("subkits." + DBConstants.EXTERNAL_KIT_NAME), @@ -634,7 +637,7 @@ public static void markKitRequestAsOrdered(ArrayList kitRequests) { SimpleResult dbVals = new SimpleResult(); try (PreparedStatement stmt = conn.prepareStatement(query)) { stmt.setLong(1, System.currentTimeMillis() / 1000); - stmt.setString(2, kitRequest.getDsmKitRequestId()); + stmt.setLong(2, kitRequest.getDsmKitRequestId()); int result = stmt.executeUpdate(); if (result != 1) { throw new RuntimeException("Error updating status as ordered"); diff --git a/src/test/java/org/broadinstitute/dsm/DDPRequestRouteTest.java b/src/test/java/org/broadinstitute/dsm/DDPRequestRouteTest.java index fe97598db..5b9718b60 100644 --- a/src/test/java/org/broadinstitute/dsm/DDPRequestRouteTest.java +++ b/src/test/java/org/broadinstitute/dsm/DDPRequestRouteTest.java @@ -105,7 +105,7 @@ public void checkForParticipant() throws Exception { response().withStatusCode(200).withBody(message.replaceAll("%1", test_participant_id).replaceAll("%2", "")) ); kitRequestTestList.add(new KitRequestShipping(test_participant_id, "TestProject_2", null, "FAKE_DSM_LABEL_UID", TEST_DDP, - "SALIVA", "1", "1", "https://easypost-files.s3-us-west-2.amazonaws.com/files/postage_label/20200214/8240f1b66535494a82b1ec0d566c3f0f.png", + "SALIVA", 1, 1, "https://easypost-files.s3-us-west-2.amazonaws.com/files/postage_label/20200214/8240f1b66535494a82b1ec0d566c3f0f.png", "", "794685038506", "9405536897846100551129", "https://track.easypost.com/djE6dHJrXzY4NGJmYzU3ZjM5OTQ1Zjg5MjEzOGRmMWVmMjI1NWZl", null, 12, false, "", 12, null, 12, "so what", "mf_testLabel", false, "shp_f470591c3fb441a68dbb9b76ecf3bb3d", 12, null, "44445", false, "NOT FOUND", null, null, null, null, 0, false, "STANDALONE", null, null, null)); @@ -120,7 +120,7 @@ public void checkForParticipant() throws Exception { response().withStatusCode(200).withBody(message.replaceAll("%1", participant_id).replaceAll("%2", Integer.toString(counter))) ); kitRequestTestList.add(new KitRequestShipping(participant_id, "TestProject_2", null, "FAKE_DSM_LABEL_UID" + counter, TEST_DDP, - "SALIVA", "1", "1", "https://easypost-files.s3-us-west-2.amazonaws.com/files/postage_label/20200214/8240f1b66535494a82b1ec0d566c3f0f.png", + "SALIVA", 1, 1, "https://easypost-files.s3-us-west-2.amazonaws.com/files/postage_label/20200214/8240f1b66535494a82b1ec0d566c3f0f.png", "", "794685038506", "9405536897846100551129", "https://track.easypost.com/djE6dHJrXzY4NGJmYzU3ZjM5OTQ1Zjg5MjEzOGRmMWVmMjI1NWZl", null, 12, false, "", 12, null, 12, "so what", "mf_testLabel", false, "shp_f470591c3fb441a68dbb9b76ecf3bb3d", 12, null, "44445", false, "NOT FOUND", null, null, null, null, 0 ,false, null, null, null, null)); @@ -148,7 +148,7 @@ public void readKitRequest() { kitRequestList.sort(new Comparator() { @Override public int compare(KitRequestShipping o1, KitRequestShipping o2) { - return Integer.parseInt(o1.getDsmKitId()) - Integer.parseInt(o2.getDsmKitId()); + return (int) (o1.getDsmKitId() - o2.getDsmKitId()); } }); for (KitRequestShipping kit : kitRequestList) { diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java index 571ffeb21..12f59a0f6 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java @@ -43,8 +43,8 @@ public void underscoresToCamelCase() { @Test public void transformObjectToMap() { Participant participant = new Participant( - "1", "QWERTY", "assigneeMr", - "assigneeTissue", "instance", "2020-10-28", + 1, "QWERTY", 0, + 0, "instance", "2020-10-28", "2020-10-28", "2020-10-28", "2020-10-28", "ptNotes", true, true, "additionalValuesJson", 1934283746283L); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/SingleProcessorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/SingleProcessorTest.java index 1c4dc8b06..7b9dd426d 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/SingleProcessorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/SingleProcessorTest.java @@ -28,7 +28,7 @@ public void processExisting() { generator.setPayload(new GeneratorPayload(new NameValue("p.additionalValuesJson", "{\"key\":\"value\"}"), 0)); ESDsm esDsm = new ESDsm(); - esDsm.setParticipant(new Participant("2174", null, null, null, null, null, null, + esDsm.setParticipant(new Participant(2174, null, 0, 0, null, null, null, null, null, null, false, false, "{\"key\": \"oldVal\"}", 12874512387612L )); BaseProcessor processor = new SingleProcessor(); processor.setEsDsm(esDsm); @@ -49,7 +49,7 @@ public void processNew() { generator.setPayload(new GeneratorPayload(new NameValue("p.additionalValuesJson", "{\"key\":\"value\"}"), 0)); ESDsm esDsm = new ESDsm(); - esDsm.setParticipant(new Participant("2174", null, null, null, null, null, null, + esDsm.setParticipant(new Participant(2174, null, 0, 0, null, null, null, null, null, null, false, false, "", 12874512387612L )); BaseProcessor processor = new SingleProcessor(); processor.setEsDsm(esDsm); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java index 30aab1fe9..53e7a9928 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java @@ -17,8 +17,8 @@ public static void setUp() { @Test public void testExport() { - final String index = "participants_structured.rgp.rgp"; - final String study = "rgp"; + final String index = "participants_structured.cmi.angio"; + final String study = "angio"; List exportables = Arrays.asList( //DynamicFieldsMappingMigrator should be first in the list to make sure that mapping will be exported for first new DynamicFieldsMappingMigrator(index, study), From b4efafc3a04385d3ffab84aa5db8a7f5b116900a Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 3 Jan 2022 17:41:57 +0400 Subject: [PATCH 282/405] DDP-7121 | ignore fields in OncHistory with the help of Jackson annotation --- src/main/java/org/broadinstitute/dsm/db/OncHistory.java | 3 +++ .../dsm/model/elastic/migration/ParticipantMigrator.java | 1 + .../elastic/migration/DynamicFieldsMappingMigratorTest.java | 1 + 3 files changed, 5 insertions(+) diff --git a/src/main/java/org/broadinstitute/dsm/db/OncHistory.java b/src/main/java/org/broadinstitute/dsm/db/OncHistory.java index 0d7936f3c..35766c7cf 100644 --- a/src/main/java/org/broadinstitute/dsm/db/OncHistory.java +++ b/src/main/java/org/broadinstitute/dsm/db/OncHistory.java @@ -1,5 +1,7 @@ package org.broadinstitute.dsm.db; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import lombok.NonNull; import org.broadinstitute.ddp.db.SimpleResult; @@ -14,6 +16,7 @@ import static org.broadinstitute.ddp.db.TransactionWrapper.inTransaction; @JsonInclude(JsonInclude.Include.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) public class OncHistory { private static final Logger logger = LoggerFactory.getLogger(OncHistory.class); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantMigrator.java index 67871a1fb..5e25be1db 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantMigrator.java @@ -15,6 +15,7 @@ public ParticipantMigrator(String index, String realm) { @Override protected Map getDataByRealm() { + System.out.println(); return (Map) Participant.getParticipants(realm); } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java index 53e7a9928..ccd9566b3 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java @@ -22,6 +22,7 @@ public void testExport() { List exportables = Arrays.asList( //DynamicFieldsMappingMigrator should be first in the list to make sure that mapping will be exported for first new DynamicFieldsMappingMigrator(index, study), + new OncHistoryMigrator(index, study), new MedicalRecordMigrator(index, study), new OncHistoryDetailsMigrator(index, study), new ParticipantDataMigrator(index, study), From 1ddee7a74341173c08c5947e5e0326558fa332d1 Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 3 Jan 2022 17:42:20 +0400 Subject: [PATCH 283/405] DDP-7121 | remove unused print --- .../dsm/model/elastic/migration/ParticipantMigrator.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantMigrator.java index 5e25be1db..6fd8b3a40 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantMigrator.java @@ -14,8 +14,5 @@ public ParticipantMigrator(String index, String realm) { } @Override - protected Map getDataByRealm() { - System.out.println(); - return (Map) Participant.getParticipants(realm); - } + protected Map getDataByRealm() { return (Map) Participant.getParticipants(realm); } } From 6b257dc47477c82b61b73efc09a144fcc076937e Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 3 Jan 2022 18:41:37 +0400 Subject: [PATCH 284/405] DDP-7121 | add numeric parsing for dynamic fields mappings --- .../dsm/model/elastic/export/parse/TypeParser.java | 4 +++- .../dsm/model/elastic/migration/DynamicFieldsTypeParser.java | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java index b49093ec1..ae369018e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java @@ -20,10 +20,12 @@ public class TypeParser extends BaseParser { public static final Map BOOLEAN_MAPPING = new HashMap<>(Map.of(MappingGenerator.TYPE, BOOLEAN)); public static final String DATE = "date"; public static final Map DATE_MAPPING = new HashMap<>(Map.of(MappingGenerator.TYPE, DATE)); + public static final String LONG = "long"; + public static final Map LONG_MAPPING = new HashMap<>(Map.of(MappingGenerator.TYPE, LONG)); @Override protected Object forNumeric(String value) { - return TEXT_KEYWORD_MAPPING; + return LONG_MAPPING; } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParser.java index e2b398955..39c12e1fa 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParser.java @@ -1,6 +1,7 @@ package org.broadinstitute.dsm.model.elastic.migration; import com.fasterxml.jackson.core.type.TypeReference; +import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; import org.broadinstitute.dsm.util.ObjectMapperSingleton; @@ -11,6 +12,8 @@ import java.util.Objects; import java.util.Optional; +import static org.broadinstitute.dsm.model.Filter.NUMBER; + class DynamicFieldsTypeParser extends TypeParser { public static final String DATE_TYPE = "DATE"; @@ -36,6 +39,8 @@ public Object parse(String type) { parsedValue = maybeType .map(this::parse) .orElse(forString(type)); + } else if (NUMBER.equals(type)) { + parsedValue = forNumeric(type); } else { parsedValue = forString(type); } From 303fd465d891926a0a51b1ea0b6647099a61a2f4 Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 4 Jan 2022 15:01:37 +0400 Subject: [PATCH 285/405] DDP-7121 | fetch field settings by instance name and realm, start writing code for generating mappings and values for dynamic fields --- .../dsm/db/dao/settings/FieldSettingsDao.java | 27 +++++++++++++++++++ .../model/elastic/export/ExportFacade.java | 6 +++-- .../elastic/export/ExportFacadePayload.java | 6 ++++- .../elastic/export/parse/BaseParser.java | 19 +++++++++++++ .../model/elastic/export/parse/Parser.java | 2 ++ .../export/parse/TypeParserFactory.java | 20 ++++++++++++++ .../DynamicFieldsMappingMigrator.java | 2 +- .../migration/DynamicFieldsTypeParser.java | 5 ++-- .../dsm/model/patch/BasePatch.java | 2 +- .../elastic/export/parse/BaseParserTest.java | 19 +++++++++++++ 10 files changed, 101 insertions(+), 7 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParserFactory.java diff --git a/src/main/java/org/broadinstitute/dsm/db/dao/settings/FieldSettingsDao.java b/src/main/java/org/broadinstitute/dsm/db/dao/settings/FieldSettingsDao.java index d7eac3b20..ef1ffb652 100644 --- a/src/main/java/org/broadinstitute/dsm/db/dao/settings/FieldSettingsDao.java +++ b/src/main/java/org/broadinstitute/dsm/db/dao/settings/FieldSettingsDao.java @@ -69,6 +69,9 @@ public class FieldSettingsDao implements Dao { private static final String SQL_DELETE_FIELD_SETTING_BY_ID = "DELETE FROM field_settings " + "WHERE field_settings_id = ?"; + private static final String SQL_DISPLAY_TYPE_BY_INSTANCE_NAME_AND_COLUMN_NAME = "SELECT display_type FROM field_settings " + + "WHERE ddp_instance_id = (select ddp_instance_id from ddp_instance where instance_name = ?) AND column_name = ?"; + private static final String BY_INSTANCE_ID = " WHERE ddp_instance_id = ?"; private static final String AND_BY_COLUMN_NAME = " AND column_name = ?"; private static final String AND_BY_COLUMN_NAMES = " AND column_name IN (?)"; @@ -155,6 +158,30 @@ public Optional get(long id) { return Optional.empty(); } + public Optional getDisplayTypeByInstanceNameAndColumnName(String instanceName, String columnName) { + SimpleResult simpleResult = inTransaction(conn -> { + SimpleResult execResult = new SimpleResult(); + try (PreparedStatement stmt = conn.prepareStatement(SQL_DISPLAY_TYPE_BY_INSTANCE_NAME_AND_COLUMN_NAME)) { + stmt.setString(1, instanceName); + stmt.setString(2, columnName); + try (ResultSet resultSet = stmt.executeQuery()) { + if (resultSet.next()) { + execResult.resultValue = resultSet.getString(DISPLAY_TYPE); + } + } + } catch (SQLException sqle) { + execResult.resultException = sqle; + } + return execResult; + }); + + if (simpleResult.resultException != null) { + throw new RuntimeException("could not find the specified display type by instance name and column name", simpleResult.resultException); + } + + return Optional.of((String) simpleResult.resultValue); + } + public List getOptionAndRadioFieldSettingsByInstanceId(int instanceId) { List fieldSettingsByOptions = new ArrayList<>(); SimpleResult results = inTransaction((conn) -> { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index b6f5f4400..d29390e3b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -14,6 +14,7 @@ import org.broadinstitute.dsm.model.elastic.export.generate.MappingGeneratorFactory; import org.broadinstitute.dsm.model.elastic.export.generate.SourceGeneratorFactory; import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; +import org.broadinstitute.dsm.model.elastic.export.parse.TypeParserFactory; import org.broadinstitute.dsm.model.elastic.export.parse.ValueParser; import org.broadinstitute.dsm.model.elastic.export.process.BaseProcessor; import org.broadinstitute.dsm.model.elastic.export.process.Processor; @@ -51,9 +52,10 @@ public void export() { private void upsertMapping() { BaseGenerator.PropertyInfo propertyInfo = getPropertyInfo(); GeneratorFactory generatorFactory = new MappingGeneratorFactory(); - propertyInfo.setFieldName(exportFacadePayload.getFieldName()); + String fieldName = exportFacadePayload.getFieldName(); + propertyInfo.setFieldName(fieldName); generator = generatorFactory.make(propertyInfo); - generator.setParser(new TypeParser()); + generator.setParser(TypeParserFactory.of(fieldName, exportFacadePayload.getRealm())); generator.setPayload(exportFacadePayload.getGeneratorPayload()); Map mappingToUpsert = generator.generate(); RequestPayload upsertMappingRequestPayload = new RequestPayload(exportFacadePayload.getIndex()); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java index 27741a59c..05d8c7e3e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java @@ -8,14 +8,16 @@ public class ExportFacadePayload { + private String realm; private String index; private String docId; private GeneratorPayload generatorPayload; - public ExportFacadePayload(String index, String docId, GeneratorPayload generatorPayload) { + public ExportFacadePayload(String index, String docId, GeneratorPayload generatorPayload, String realm) { this.index = Objects.requireNonNull(index); this.docId = Objects.requireNonNull(docId); this.generatorPayload = Objects.requireNonNull(generatorPayload); + this.realm = Objects.requireNonNull(realm); } public String getIndex() { @@ -50,4 +52,6 @@ public String getFieldNameWithAlias() { public String getFieldName() { return generatorPayload.getFieldName(); } + + public String getRealm() { return realm; } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java index 252b7f263..30cb4d1a1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java @@ -1,14 +1,33 @@ package org.broadinstitute.dsm.model.elastic.export.parse; import org.apache.commons.lang3.StringUtils; +import org.broadinstitute.dsm.db.dao.settings.FieldSettingsDao; +import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; import java.time.format.DateTimeParseException; +import java.util.Optional; public abstract class BaseParser implements Parser { + protected BaseGenerator.PropertyInfo propertyInfo; + protected String fieldName; + protected String realm; + + public void setPropertyInfo(BaseGenerator.PropertyInfo propertyInfo) { + this.propertyInfo = propertyInfo; + } + + public void setFieldName(String fieldName) { + this.fieldName = fieldName; + } + + public void setRealm(String realm) { + this.realm = realm; + } + @Override public Object parse(String value) { value = convertString(value); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/Parser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/Parser.java index 996b985a7..b590067d4 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/Parser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/Parser.java @@ -4,6 +4,8 @@ public interface Parser { Object parse(String value); + Object parse(); + default Object[] parse(String[] values) { throw new UnsupportedOperationException(); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParserFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParserFactory.java new file mode 100644 index 000000000..fa0776ceb --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParserFactory.java @@ -0,0 +1,20 @@ +package org.broadinstitute.dsm.model.elastic.export.parse; + +import org.broadinstitute.dsm.db.dao.settings.FieldSettingsDao; +import org.broadinstitute.dsm.model.elastic.migration.DynamicFieldsTypeParser; +import org.broadinstitute.dsm.statics.ESObjectConstants; + +import java.util.Optional; + +public class TypeParserFactory { + + public static TypeParser of(String fieldName, String realm) { + TypeParser typeParser = new TypeParser(); + if (ESObjectConstants.ADDITIONAL_VALUES_JSON.equals(fieldName) || ESObjectConstants.DATA.equals(fieldName)) { + Optional displayTypeByInstanceNameAndColumnName = FieldSettingsDao.of().getDisplayTypeByInstanceNameAndColumnName(realm, fieldName); + if (displayTypeByInstanceNameAndColumnName.isPresent()) + typeParser = new DynamicFieldsTypeParser(); + } + return typeParser; + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java index 336116652..2e5d5dcea 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java @@ -63,7 +63,7 @@ private Map buildFinalMapping() { private void buildMapping(FieldSettingsDto fieldSettingsDto, BaseGenerator.PropertyInfo propertyInfo) { String columnName = Util.underscoresToCamelCase(fieldSettingsDto.getColumnName()); String propertyName = propertyInfo.getPropertyName(); - Object typeMap = parser.parse(fieldSettingsDto.getDisplayType()); + Object typeMap = parser.parse(); if (!(propertyMap.containsKey(propertyName))) { Map dynamicFields = new HashMap<>(Map.of(DYNAMIC_FIELDS_WRAPPER_NAME, new HashMap<>(Map.of(PROPERTIES, new HashMap<>(Map.of(columnName, typeMap)))))); Map wrapperMap = new HashMap<>(); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParser.java index 39c12e1fa..7a647ec8a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParser.java @@ -14,7 +14,7 @@ import static org.broadinstitute.dsm.model.Filter.NUMBER; -class DynamicFieldsTypeParser extends TypeParser { +public class DynamicFieldsTypeParser extends TypeParser { public static final String DATE_TYPE = "DATE"; public static final String CHECKBOX_TYPE = "CHECKBOX"; @@ -27,7 +27,8 @@ public void setFieldSettingsDto(FieldSettingsDto fieldSettingsDto) { } @Override - public Object parse(String type) { + public Object parse() { + String type = fieldSettingsDto.getDisplayType(); Object parsedValue; if (DATE_TYPE.equals(type)) { parsedValue = forDate(type); diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index 59c3b085f..1679fe3b7 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -68,7 +68,7 @@ protected BasePatch(Patch patch) { private void exportToES(NameValue nameValue) { GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, Integer.parseInt(patch.getId()), patch.getParent(), patch.getParentId()); ExportFacadePayload exportFacadePayload = - new ExportFacadePayload(ddpInstance.getParticipantIndexES(), patch.getDdpParticipantId(), generatorPayload); + new ExportFacadePayload(ddpInstance.getParticipantIndexES(), patch.getDdpParticipantId(), generatorPayload, patch.getRealm()); ExportFacade exportFacade = new ExportFacade(exportFacadePayload); exportFacade.export(); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserTest.java index c0baeeb32..f3c7aae69 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserTest.java @@ -1,7 +1,10 @@ package org.broadinstitute.dsm.model.elastic.export.parse; +import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.TEXT_KEYWORD_MAPPING; import static org.junit.Assert.*; +import org.broadinstitute.dsm.db.MedicalRecord; +import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; @@ -37,4 +40,20 @@ public void convertString() { Assert.assertEquals("2015-01-01", convertedValue2); } + @Test + public void parse() { + + BaseGenerator.PropertyInfo propertyInfo = new BaseGenerator.PropertyInfo(MedicalRecord.class, true); + propertyInfo.setFieldName("additionalValuesJson"); + + BaseParser typeParser = new TypeParser(); + typeParser.setPropertyInfo(propertyInfo); + typeParser.setFieldName("scooby"); + + Object parsedObject = typeParser.parse("10"); + + Assert.assertEquals(TEXT_KEYWORD_MAPPING, parsedObject); + + } + } \ No newline at end of file From 10c525dd19b744e38d000c734ddbd41ade683e33 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 4 Jan 2022 16:41:02 +0400 Subject: [PATCH 286/405] DDP-7121 | change parser for dynamic fields --- .../model/elastic/export/ExportFacade.java | 8 +--- .../elastic/export/ExportFacadePayload.java | 12 ++++- .../export/generate/BaseGenerator.java | 2 +- .../export/generate/GeneratorPayload.java | 6 ++- .../elastic/export/parse/BaseParser.java | 8 ++-- .../model/elastic/export/parse/Parser.java | 2 +- .../elastic/export/parse/TypeParser.java | 2 - .../export/parse/TypeParserFactory.java | 15 ++++-- .../DynamicFieldsMappingMigrator.java | 3 +- .../migration/DynamicFieldsTypeParser.java | 48 ++++++++++--------- .../elastic/export/parse/BaseParserTest.java | 16 ------- .../elastic/export/parse/TypeParserTest.java | 41 ++++++++++++++++ .../DynamicFieldsTypeParserTest.java | 7 +-- 13 files changed, 103 insertions(+), 67 deletions(-) create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParserTest.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index d29390e3b..f7633919a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -1,7 +1,6 @@ package org.broadinstitute.dsm.model.elastic.export; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Objects; @@ -13,14 +12,11 @@ import org.broadinstitute.dsm.model.elastic.export.generate.MappingGenerator; import org.broadinstitute.dsm.model.elastic.export.generate.MappingGeneratorFactory; import org.broadinstitute.dsm.model.elastic.export.generate.SourceGeneratorFactory; -import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; import org.broadinstitute.dsm.model.elastic.export.parse.TypeParserFactory; import org.broadinstitute.dsm.model.elastic.export.parse.ValueParser; import org.broadinstitute.dsm.model.elastic.export.process.BaseProcessor; -import org.broadinstitute.dsm.model.elastic.export.process.Processor; import org.broadinstitute.dsm.model.elastic.export.process.ProcessorFactory; import org.broadinstitute.dsm.model.elastic.export.process.ProcessorFactoryImpl; -import org.broadinstitute.dsm.model.elastic.export.process.SingleProcessor; import org.broadinstitute.dsm.model.elastic.search.DefaultDeserializer; import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; @@ -52,10 +48,10 @@ public void export() { private void upsertMapping() { BaseGenerator.PropertyInfo propertyInfo = getPropertyInfo(); GeneratorFactory generatorFactory = new MappingGeneratorFactory(); - String fieldName = exportFacadePayload.getFieldName(); + String fieldName = exportFacadePayload.getCamelCaseFieldName(); propertyInfo.setFieldName(fieldName); generator = generatorFactory.make(propertyInfo); - generator.setParser(TypeParserFactory.of(fieldName, exportFacadePayload.getRealm())); + generator.setParser(TypeParserFactory.of(exportFacadePayload)); generator.setPayload(exportFacadePayload.getGeneratorPayload()); Map mappingToUpsert = generator.generate(); RequestPayload upsertMappingRequestPayload = new RequestPayload(exportFacadePayload.getIndex()); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java index 05d8c7e3e..33cc3292c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacadePayload.java @@ -49,8 +49,16 @@ public String getFieldNameWithAlias() { return generatorPayload.getName(); } - public String getFieldName() { - return generatorPayload.getFieldName(); + public String getCamelCaseFieldName() { + return generatorPayload.getCamelCaseFieldName(); + } + + public String getRawFieldName() { + return generatorPayload.getRawFieldName(); + } + + public Object getValue() { + return generatorPayload.getValue(); } public String getRealm() { return realm; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java index 69be19655..964652b57 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java @@ -72,7 +72,7 @@ public String getPropertyName() { } public String getFieldName() { - return generatorPayload.getFieldName(); + return generatorPayload.getCamelCaseFieldName(); } @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorPayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorPayload.java index 663612ff7..08be74550 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorPayload.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorPayload.java @@ -49,7 +49,11 @@ public String getParentId() { return parentId; } - public String getFieldName() { + public String getCamelCaseFieldName() { return Util.underscoresToCamelCase(Util.getDBElement(getName()).getColumnName()); } + + public String getRawFieldName() { + return Util.getDBElement(getName()).getColumnName(); + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java index 30cb4d1a1..480a4c16b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java @@ -1,14 +1,12 @@ package org.broadinstitute.dsm.model.elastic.export.parse; -import org.apache.commons.lang3.StringUtils; -import org.broadinstitute.dsm.db.dao.settings.FieldSettingsDao; -import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; - import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; import java.time.format.DateTimeParseException; -import java.util.Optional; + +import org.apache.commons.lang3.StringUtils; +import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; public abstract class BaseParser implements Parser { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/Parser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/Parser.java index b590067d4..823b4f6ac 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/Parser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/Parser.java @@ -4,7 +4,7 @@ public interface Parser { Object parse(String value); - Object parse(); + default Object parse() { throw new UnsupportedOperationException(); } default Object[] parse(String[] values) { throw new UnsupportedOperationException(); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java index ae369018e..80df8702a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java @@ -43,6 +43,4 @@ protected Object forString(String value) { return TEXT_KEYWORD_MAPPING; } - - } \ No newline at end of file diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParserFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParserFactory.java index fa0776ceb..f7d020c25 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParserFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParserFactory.java @@ -1,6 +1,7 @@ package org.broadinstitute.dsm.model.elastic.export.parse; import org.broadinstitute.dsm.db.dao.settings.FieldSettingsDao; +import org.broadinstitute.dsm.model.elastic.export.ExportFacadePayload; import org.broadinstitute.dsm.model.elastic.migration.DynamicFieldsTypeParser; import org.broadinstitute.dsm.statics.ESObjectConstants; @@ -8,13 +9,17 @@ public class TypeParserFactory { - public static TypeParser of(String fieldName, String realm) { + public static TypeParser of(ExportFacadePayload exportFacadePayload) { TypeParser typeParser = new TypeParser(); - if (ESObjectConstants.ADDITIONAL_VALUES_JSON.equals(fieldName) || ESObjectConstants.DATA.equals(fieldName)) { - Optional displayTypeByInstanceNameAndColumnName = FieldSettingsDao.of().getDisplayTypeByInstanceNameAndColumnName(realm, fieldName); - if (displayTypeByInstanceNameAndColumnName.isPresent()) - typeParser = new DynamicFieldsTypeParser(); + if (isDynamicFields(exportFacadePayload.getCamelCaseFieldName())) { + DynamicFieldsTypeParser dynamicFieldsTypeParser = new DynamicFieldsTypeParser(); + dynamicFieldsTypeParser.setRealm(exportFacadePayload.getRealm()); + typeParser = dynamicFieldsTypeParser; } return typeParser; } + + private static boolean isDynamicFields(String fieldName) { + return ESObjectConstants.ADDITIONAL_VALUES_JSON.equals(fieldName) || ESObjectConstants.DATA.equals(fieldName); + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java index 2e5d5dcea..ecbe699a9 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java @@ -40,7 +40,8 @@ public void export() { FieldSettingsDao fieldSettingsDao = FieldSettingsDao.of(); List fieldSettingsByStudyName = fieldSettingsDao.getAllFieldSettings(); for (FieldSettingsDto fieldSettingsDto : fieldSettingsByStudyName) { - parser.setFieldSettingsDto(fieldSettingsDto); + parser.setDisplayType(fieldSettingsDto.getDisplayType()); + parser.setPossibleValuesJson(fieldSettingsDto.getPossibleValues()); String fieldType = fieldSettingsDto.getFieldType(); BaseGenerator.PropertyInfo propertyInfo = Util.TABLE_ALIAS_MAPPINGS.get(fieldType); if (propertyInfo != null) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParser.java index 7a647ec8a..cbde5e334 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParser.java @@ -2,14 +2,13 @@ import com.fasterxml.jackson.core.type.TypeReference; import org.apache.commons.lang3.StringUtils; -import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; +import org.broadinstitute.dsm.db.dao.settings.FieldSettingsDao; import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; import org.broadinstitute.dsm.util.ObjectMapperSingleton; import java.io.IOException; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import static org.broadinstitute.dsm.model.Filter.NUMBER; @@ -20,35 +19,40 @@ public class DynamicFieldsTypeParser extends TypeParser { public static final String CHECKBOX_TYPE = "CHECKBOX"; public static final String ACTIVITY_STAFF_TYPE = "ACTIVITY_STAFF"; public static final String ACTIVITY_TYPE = "ACTIVITY"; - private FieldSettingsDto fieldSettingsDto; + private String displayType; + private String possibleValuesJson; - public void setFieldSettingsDto(FieldSettingsDto fieldSettingsDto) { - this.fieldSettingsDto = fieldSettingsDto; + public void setDisplayType(String displayType) { + this.displayType = displayType; + } + + public void setPossibleValuesJson(String possibleValuesJson) { + this.possibleValuesJson = possibleValuesJson; } @Override - public Object parse() { - String type = fieldSettingsDto.getDisplayType(); + public Object parse(String fieldName) { + displayType = FieldSettingsDao.of().getDisplayTypeByInstanceNameAndColumnName(realm, fieldName).orElse(StringUtils.EMPTY); Object parsedValue; - if (DATE_TYPE.equals(type)) { - parsedValue = forDate(type); - } else if (CHECKBOX_TYPE.equals(type)) { - parsedValue = forBoolean(type); - } else if (isActivityRelatedType(type)) { - String possibleValuesJson = Objects.requireNonNull(fieldSettingsDto).getPossibleValues(); - Optional maybeType = getTypeFromPossibleValuesJson(possibleValuesJson); + if (DATE_TYPE.equals(displayType)) { + parsedValue = forDate(displayType); + } else if (CHECKBOX_TYPE.equals(displayType)) { + parsedValue = forBoolean(displayType); + } else if (isActivityRelatedType()) { + Optional maybeType = getTypeFromPossibleValuesJson(); + this.displayType = maybeType.orElse(StringUtils.EMPTY); parsedValue = maybeType .map(this::parse) - .orElse(forString(type)); - } else if (NUMBER.equals(type)) { - parsedValue = forNumeric(type); + .orElse(forString(displayType)); + } else if (NUMBER.equals(displayType)) { + parsedValue = forNumeric(displayType); } else { - parsedValue = forString(type); + parsedValue = forString(displayType); } return parsedValue; } - private Optional getTypeFromPossibleValuesJson(String possibleValuesJson) { + private Optional getTypeFromPossibleValuesJson() { try { List> possibleValues = ObjectMapperSingleton.instance().readValue(possibleValuesJson, new TypeReference>>() {}); Optional maybeType = possibleValues.stream() @@ -57,11 +61,11 @@ private Optional getTypeFromPossibleValuesJson(String possibleValuesJson .findFirst(); return maybeType; } catch (IOException e) { - throw new RuntimeException(e); + return Optional.empty(); } } - private boolean isActivityRelatedType(String type) { - return ACTIVITY_STAFF_TYPE.equals(type) || ACTIVITY_TYPE.equals(type); + private boolean isActivityRelatedType() { + return ACTIVITY_STAFF_TYPE.equals(displayType) || ACTIVITY_TYPE.equals(displayType); } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserTest.java index f3c7aae69..e3d7df04b 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserTest.java @@ -40,20 +40,4 @@ public void convertString() { Assert.assertEquals("2015-01-01", convertedValue2); } - @Test - public void parse() { - - BaseGenerator.PropertyInfo propertyInfo = new BaseGenerator.PropertyInfo(MedicalRecord.class, true); - propertyInfo.setFieldName("additionalValuesJson"); - - BaseParser typeParser = new TypeParser(); - typeParser.setPropertyInfo(propertyInfo); - typeParser.setFieldName("scooby"); - - Object parsedObject = typeParser.parse("10"); - - Assert.assertEquals(TEXT_KEYWORD_MAPPING, parsedObject); - - } - } \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParserTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParserTest.java new file mode 100644 index 000000000..c64eed6cd --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParserTest.java @@ -0,0 +1,41 @@ +package org.broadinstitute.dsm.model.elastic.export.parse; + +import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.*; +import static org.junit.Assert.*; + +import org.broadinstitute.dsm.db.MedicalRecord; +import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; +import org.broadinstitute.dsm.model.elastic.migration.DynamicFieldsTypeParser; +import org.junit.Assert; +import org.junit.Test; + +public class TypeParserTest { + + + @Test + public void parse() { + + BaseGenerator.PropertyInfo propertyInfo = new BaseGenerator.PropertyInfo(MedicalRecord.class, true); + propertyInfo.setFieldName("additionalValuesJson"); + + DynamicFieldsTypeParser typeParser = new DynamicFieldsTypeParser(); + typeParser.setPropertyInfo(propertyInfo); + typeParser.setFieldName("scooby"); + typeParser.setDisplayType("TEXT"); + Object parsedObject = typeParser.parse(); + Assert.assertEquals(TEXT_KEYWORD_MAPPING, parsedObject); + + typeParser.setDisplayType("NUMBER"); + Object parsedObject1 = typeParser.parse(); + Assert.assertEquals(LONG_MAPPING, parsedObject1); + + typeParser.setDisplayType("DATE"); + Object parsedObject2 = typeParser.parse(); + Assert.assertEquals(DATE_MAPPING, parsedObject2); + + typeParser.setDisplayType("CHECKBOX"); + Object parsedObject3 = typeParser.parse(); + Assert.assertEquals(BOOLEAN_MAPPING, parsedObject3); + } + +} \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParserTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParserTest.java index 0e0d8d8bf..f1fa7bb5f 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParserTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParserTest.java @@ -15,12 +15,9 @@ public class DynamicFieldsTypeParserTest { public void parse() { String possibleValuesJson = "[{\"value\":\"CONSENT.completedAt\",\"type\":\"DATE\"}]"; String displayType = "ACTIVITY_STAFF"; - FieldSettingsDto fieldSettingsDto = new FieldSettingsDto.Builder(0) - .withDisplayType(displayType) - .withPossibleValues(possibleValuesJson) - .build(); DynamicFieldsTypeParser dynamicFieldsTypeParser = new DynamicFieldsTypeParser(); - dynamicFieldsTypeParser.setFieldSettingsDto(fieldSettingsDto); + dynamicFieldsTypeParser.setDisplayType(displayType); + dynamicFieldsTypeParser.setPossibleValuesJson(possibleValuesJson); Map mapping = (Map)dynamicFieldsTypeParser.parse(displayType); Object date = mapping.get(MappingGenerator.TYPE); assertEquals(TypeParser.DATE, date); From c85caf1d6576feceec73dfad7a4a695de1d5c868 Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 4 Jan 2022 17:13:22 +0400 Subject: [PATCH 287/405] DDP-7121 | correct Parser interface, implement parsing for dynamic and non-dynamic fields --- .../model/elastic/export/generate/MappingGenerator.java | 2 +- .../dsm/model/elastic/export/parse/Parser.java | 3 --- .../elastic/migration/DynamicFieldsMappingMigrator.java | 2 +- .../model/elastic/migration/DynamicFieldsTypeParser.java | 8 +++++++- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java index 6e690f8ee..2e1a30866 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java @@ -42,7 +42,7 @@ protected Map parseJson() { Map resultMap = new HashMap<>(); Map fieldsByValues = parseJsonToMapFromValue(); for (Map.Entry entry: fieldsByValues.entrySet()) { - Object eachType = parser.parse(String.valueOf(entry.getValue())); + Object eachType = parser.parse(entry.getKey()); resultMap.put(Util.underscoresToCamelCase(entry.getKey()), eachType); } Map returnMap = new HashMap<>(Map.of(ESObjectConstants.DYNAMIC_FIELDS, new HashMap<>(Map.of(PROPERTIES, resultMap)))); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/Parser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/Parser.java index 823b4f6ac..9565f2687 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/Parser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/Parser.java @@ -3,9 +3,6 @@ public interface Parser { Object parse(String value); - - default Object parse() { throw new UnsupportedOperationException(); } - default Object[] parse(String[] values) { throw new UnsupportedOperationException(); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java index ecbe699a9..9f707f963 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java @@ -64,7 +64,7 @@ private Map buildFinalMapping() { private void buildMapping(FieldSettingsDto fieldSettingsDto, BaseGenerator.PropertyInfo propertyInfo) { String columnName = Util.underscoresToCamelCase(fieldSettingsDto.getColumnName()); String propertyName = propertyInfo.getPropertyName(); - Object typeMap = parser.parse(); + Object typeMap = parser.parse(fieldSettingsDto.getDisplayType()); if (!(propertyMap.containsKey(propertyName))) { Map dynamicFields = new HashMap<>(Map.of(DYNAMIC_FIELDS_WRAPPER_NAME, new HashMap<>(Map.of(PROPERTIES, new HashMap<>(Map.of(columnName, typeMap)))))); Map wrapperMap = new HashMap<>(); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParser.java index cbde5e334..df760b559 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParser.java @@ -32,7 +32,10 @@ public void setPossibleValuesJson(String possibleValuesJson) { @Override public Object parse(String fieldName) { - displayType = FieldSettingsDao.of().getDisplayTypeByInstanceNameAndColumnName(realm, fieldName).orElse(StringUtils.EMPTY); + + if (StringUtils.isBlank(displayType)) + displayType = FieldSettingsDao.of().getDisplayTypeByInstanceNameAndColumnName(realm, fieldName).orElse(StringUtils.EMPTY); + Object parsedValue; if (DATE_TYPE.equals(displayType)) { parsedValue = forDate(displayType); @@ -49,6 +52,9 @@ public Object parse(String fieldName) { } else { parsedValue = forString(displayType); } + + displayType = null; + return parsedValue; } From 63bf035994669f35277b689f258d700d016332b6 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 4 Jan 2022 17:41:05 +0400 Subject: [PATCH 288/405] DDP-7121 | start overriding of method parse of type parser --- .../elastic/export/parse/TypeParser.java | 20 ++++++++++++++++ .../elastic/export/parse/TypeParserTest.java | 24 +++++++++++++++---- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java index 80df8702a..7687b51b6 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java @@ -1,5 +1,6 @@ package org.broadinstitute.dsm.model.elastic.export.parse; +import java.lang.reflect.Field; import java.util.HashMap; import java.util.Map; @@ -23,6 +24,25 @@ public class TypeParser extends BaseParser { public static final String LONG = "long"; public static final Map LONG_MAPPING = new HashMap<>(Map.of(MappingGenerator.TYPE, LONG)); + @Override + public Object parse(String fieldName) { + Class propertyClass = propertyInfo.getPropertyClass(); + Object mappingType; + try { + Field field = propertyClass.getField(fieldName); + if (long.class.isAssignableFrom(field.getType())) { + mappingType = LONG_MAPPING; + } else if (boolean.class.isAssignableFrom(field.getType())) { + mappingType = BOOLEAN_MAPPING; + } else { + // either text or date in string + } + } catch (NoSuchFieldException e) { + throw new RuntimeException(e); + } + return super.parse(fieldName); + } + @Override protected Object forNumeric(String value) { return LONG_MAPPING; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParserTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParserTest.java index c64eed6cd..580561a4d 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParserTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParserTest.java @@ -3,8 +3,12 @@ import static org.broadinstitute.dsm.model.elastic.export.parse.TypeParser.*; import static org.junit.Assert.*; +import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.MedicalRecord; +import org.broadinstitute.dsm.model.NameValue; +import org.broadinstitute.dsm.model.elastic.export.ExportFacadePayload; import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; +import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; import org.broadinstitute.dsm.model.elastic.migration.DynamicFieldsTypeParser; import org.junit.Assert; import org.junit.Test; @@ -13,7 +17,7 @@ public class TypeParserTest { @Test - public void parse() { + public void parseDynamicFields() { BaseGenerator.PropertyInfo propertyInfo = new BaseGenerator.PropertyInfo(MedicalRecord.class, true); propertyInfo.setFieldName("additionalValuesJson"); @@ -22,20 +26,30 @@ public void parse() { typeParser.setPropertyInfo(propertyInfo); typeParser.setFieldName("scooby"); typeParser.setDisplayType("TEXT"); - Object parsedObject = typeParser.parse(); + Object parsedObject = typeParser.parse(StringUtils.EMPTY); Assert.assertEquals(TEXT_KEYWORD_MAPPING, parsedObject); typeParser.setDisplayType("NUMBER"); - Object parsedObject1 = typeParser.parse(); + Object parsedObject1 = typeParser.parse(StringUtils.EMPTY); Assert.assertEquals(LONG_MAPPING, parsedObject1); typeParser.setDisplayType("DATE"); - Object parsedObject2 = typeParser.parse(); + Object parsedObject2 = typeParser.parse(StringUtils.EMPTY); Assert.assertEquals(DATE_MAPPING, parsedObject2); typeParser.setDisplayType("CHECKBOX"); - Object parsedObject3 = typeParser.parse(); + Object parsedObject3 = typeParser.parse(StringUtils.EMPTY); Assert.assertEquals(BOOLEAN_MAPPING, parsedObject3); } + @Test + public void parseNonDynamicFields() { + NameValue crRequired = new NameValue("crRequired", true); + GeneratorPayload generatorPayload = new GeneratorPayload(crRequired); + ExportFacadePayload exportFacadePayload = new ExportFacadePayload("", "", generatorPayload, ""); + TypeParser typeParser = TypeParserFactory.of(exportFacadePayload); + Object booleanMapping = typeParser.parse("true"); + assertEquals(BOOLEAN_MAPPING, booleanMapping); + } + } \ No newline at end of file From 7db0621777a8d7304da1a3f9490400f78111ef36 Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 4 Jan 2022 19:06:07 +0400 Subject: [PATCH 289/405] DDP-7121 | add Date-specific annotations to ES classes, start refactoring type and value parsers --- .../org/broadinstitute/dsm/db/OncHistory.java | 6 ++++ .../dsm/db/OncHistoryDetail.java | 1 + .../broadinstitute/dsm/db/Participant.java | 4 +++ .../org/broadinstitute/dsm/db/Tissue.java | 4 ++- .../model/elastic/export/ExportFacade.java | 5 ++- .../export/generate/BaseGenerator.java | 6 ++-- .../export/generate/MappingGenerator.java | 5 +++ .../export/generate/SourceGenerator.java | 5 +++ .../elastic/export/parse/BaseParser.java | 32 +++++++++++++------ .../elastic/export/parse/TypeParser.java | 21 ++---------- .../elastic/export/parse/TypeParserTest.java | 16 ++++++++-- 11 files changed, 70 insertions(+), 35 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/OncHistory.java b/src/main/java/org/broadinstitute/dsm/db/OncHistory.java index 35766c7cf..44149bb83 100644 --- a/src/main/java/org/broadinstitute/dsm/db/OncHistory.java +++ b/src/main/java/org/broadinstitute/dsm/db/OncHistory.java @@ -5,6 +5,8 @@ import com.fasterxml.jackson.annotation.JsonInclude; import lombok.NonNull; import org.broadinstitute.ddp.db.SimpleResult; +import org.broadinstitute.dsm.db.structure.DbDateConversion; +import org.broadinstitute.dsm.db.structure.SqlDateConverter; import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.util.SystemUtil; import org.slf4j.Logger; @@ -24,7 +26,11 @@ public class OncHistory { private static final String SQL_UPDATE_ONC_HISTORY = "UPDATE ddp_onc_history SET created = ?, last_changed = ?, changed_by = ? WHERE participant_id = ? AND created IS NULL"; private long participantId; + + @DbDateConversion(SqlDateConverter.STRING_DAY) private String created; + + @DbDateConversion(SqlDateConverter.STRING_DAY) private String reviewed; private String changedBy; diff --git a/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java b/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java index b65d958c7..982d4ce44 100644 --- a/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java +++ b/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java @@ -93,6 +93,7 @@ public class OncHistoryDetail { private long medicalRecordId; @ColumnName (DBConstants.DATE_PX) + @DbDateConversion(SqlDateConverter.STRING_DAY) private String datePx; @ColumnName (DBConstants.TYPE_PX) diff --git a/src/main/java/org/broadinstitute/dsm/db/Participant.java b/src/main/java/org/broadinstitute/dsm/db/Participant.java index 35371ae44..f7c86109b 100644 --- a/src/main/java/org/broadinstitute/dsm/db/Participant.java +++ b/src/main/java/org/broadinstitute/dsm/db/Participant.java @@ -10,6 +10,8 @@ import org.apache.commons.lang3.StringUtils; import org.broadinstitute.ddp.db.SimpleResult; import org.broadinstitute.dsm.db.structure.ColumnName; +import org.broadinstitute.dsm.db.structure.DbDateConversion; +import org.broadinstitute.dsm.db.structure.SqlDateConverter; import org.broadinstitute.dsm.db.structure.TableName; import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.util.DBUtil; @@ -87,6 +89,7 @@ public class Participant { primaryKey = DBConstants.PARTICIPANT_ID, columnPrefix = "") @ColumnName (DBConstants.CR_SENT) + @DbDateConversion(SqlDateConverter.STRING_DAY) private String crSent; @TableName ( @@ -95,6 +98,7 @@ public class Participant { primaryKey = DBConstants.PARTICIPANT_ID, columnPrefix = "") @ColumnName (DBConstants.CR_RECEIVED) + @DbDateConversion(SqlDateConverter.STRING_DAY) private String crReceived; @TableName ( diff --git a/src/main/java/org/broadinstitute/dsm/db/Tissue.java b/src/main/java/org/broadinstitute/dsm/db/Tissue.java index 7646c7fa1..167c8bfc1 100644 --- a/src/main/java/org/broadinstitute/dsm/db/Tissue.java +++ b/src/main/java/org/broadinstitute/dsm/db/Tissue.java @@ -85,6 +85,7 @@ public class Tissue { private String collaboratorSampleId; @ColumnName (DBConstants.BLOCK_SENT) + @DbDateConversion(SqlDateConverter.STRING_DAY) private String blockSent; @ColumnName (DBConstants.SHL_WORK_NUMBER) @@ -129,11 +130,12 @@ public Map getDynamicFields() { @ColumnName (DBConstants.TISSUE_RETURN_DATE) @DbDateConversion(SqlDateConverter.STRING_DAY) private String returnDate; - // + @ColumnName (DBConstants.RETURN_FEDEX_ID) private String returnFedexId; @ColumnName (DBConstants.EXPECTED_RETURN) + @DbDateConversion(SqlDateConverter.STRING_DAY) private String expectedReturn; @ColumnName (DBConstants.TUMOR_PERCENTAGE) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index f7633919a..7539ecbd3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -12,6 +12,7 @@ import org.broadinstitute.dsm.model.elastic.export.generate.MappingGenerator; import org.broadinstitute.dsm.model.elastic.export.generate.MappingGeneratorFactory; import org.broadinstitute.dsm.model.elastic.export.generate.SourceGeneratorFactory; +import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; import org.broadinstitute.dsm.model.elastic.export.parse.TypeParserFactory; import org.broadinstitute.dsm.model.elastic.export.parse.ValueParser; import org.broadinstitute.dsm.model.elastic.export.process.BaseProcessor; @@ -51,7 +52,9 @@ private void upsertMapping() { String fieldName = exportFacadePayload.getCamelCaseFieldName(); propertyInfo.setFieldName(fieldName); generator = generatorFactory.make(propertyInfo); - generator.setParser(TypeParserFactory.of(exportFacadePayload)); + TypeParser typeParser = TypeParserFactory.of(exportFacadePayload); + typeParser.setPropertyInfo(propertyInfo); + generator.setParser(typeParser); generator.setPayload(exportFacadePayload.getGeneratorPayload()); Map mappingToUpsert = generator.generate(); RequestPayload upsertMappingRequestPayload = new RequestPayload(exportFacadePayload.getIndex()); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java index 964652b57..d66f575c0 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java @@ -93,9 +93,11 @@ protected Map parseJsonToMapFromValue() { } protected Object parseSingleElement() { - Object element = parser.parse(String.valueOf(getNameValue().getValue())); + Object element = parseElement(); return getElement(element); - }; + } + + protected abstract Object parseElement(); protected abstract Object getElement(Object type); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java index 2e1a30866..d1cfb221a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java @@ -29,6 +29,11 @@ public Map generate() { return getCompleteMap(construct()); } + @Override + protected Object parseElement() { + return parser.parse(getFieldName()); + } + public Map getCompleteMap(Object propertyMap) { String propertyName = getPropertyName(); Map objectLevel = new HashMap<>(Map.of(propertyName, propertyMap)); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java index d1e47529e..fbcaaba8b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java @@ -30,6 +30,11 @@ private Map buildPropertyLevelWithData(Object dataToExport) { return new HashMap<>(Map.of(getPropertyName(), dataToExport)); } + @Override + protected Object parseElement() { + return parser.parse(String.valueOf(getNameValue().getValue())); + } + @Override protected Object parseJson() { return construct(); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java index 480a4c16b..3d72009d9 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java @@ -1,11 +1,13 @@ package org.broadinstitute.dsm.model.elastic.export.parse; +import java.lang.reflect.Field; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; import java.time.format.DateTimeParseException; import org.apache.commons.lang3.StringUtils; +import org.broadinstitute.dsm.db.structure.DbDateConversion; import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; public abstract class BaseParser implements Parser { @@ -27,17 +29,27 @@ public void setRealm(String realm) { } @Override - public Object parse(String value) { - value = convertString(value); - Object result = forString(value); - if (StringUtils.isNumeric(value)) { - result = forNumeric(value); - } else if (isBoolean(value)) { - result = forBoolean(convertBoolean(value)); - } else if (isDateOrTimeOrDateTime(value)) { - result = forDate(value); + public Object parse(String element) { + Class propertyClass = propertyInfo.getPropertyClass(); + Object elementMap; + try { + Field field = propertyClass.getDeclaredField(element); + if (long.class.isAssignableFrom(field.getType())) { + elementMap = forNumeric(element); + } else if (boolean.class.isAssignableFrom(field.getType())) { + elementMap = forBoolean(element); + } else { + // either text or date in string + if (field.getAnnotation(DbDateConversion.class) != null) { + elementMap = forDate(element); + } else { + elementMap = forString(element); + } + } + } catch (NoSuchFieldException e) { + throw new RuntimeException(e); } - return result; + return elementMap; } protected abstract Object forNumeric(String value); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java index 7687b51b6..cee297e01 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java @@ -4,7 +4,9 @@ import java.util.HashMap; import java.util.Map; +import org.broadinstitute.dsm.db.structure.DbDateConversion; import org.broadinstitute.dsm.model.elastic.export.generate.MappingGenerator; +import org.elasticsearch.common.text.Text; public class TypeParser extends BaseParser { @@ -24,25 +26,6 @@ public class TypeParser extends BaseParser { public static final String LONG = "long"; public static final Map LONG_MAPPING = new HashMap<>(Map.of(MappingGenerator.TYPE, LONG)); - @Override - public Object parse(String fieldName) { - Class propertyClass = propertyInfo.getPropertyClass(); - Object mappingType; - try { - Field field = propertyClass.getField(fieldName); - if (long.class.isAssignableFrom(field.getType())) { - mappingType = LONG_MAPPING; - } else if (boolean.class.isAssignableFrom(field.getType())) { - mappingType = BOOLEAN_MAPPING; - } else { - // either text or date in string - } - } catch (NoSuchFieldException e) { - throw new RuntimeException(e); - } - return super.parse(fieldName); - } - @Override protected Object forNumeric(String value) { return LONG_MAPPING; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParserTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParserTest.java index 580561a4d..754c3ec01 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParserTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParserTest.java @@ -44,12 +44,24 @@ public void parseDynamicFields() { @Test public void parseNonDynamicFields() { - NameValue crRequired = new NameValue("crRequired", true); + NameValue crRequired = new NameValue("m.crRequired", true); GeneratorPayload generatorPayload = new GeneratorPayload(crRequired); ExportFacadePayload exportFacadePayload = new ExportFacadePayload("", "", generatorPayload, ""); TypeParser typeParser = TypeParserFactory.of(exportFacadePayload); - Object booleanMapping = typeParser.parse("true"); + typeParser.setPropertyInfo(new BaseGenerator.PropertyInfo(MedicalRecord.class, true)); + Object booleanMapping = typeParser.parse("crRequired"); assertEquals(BOOLEAN_MAPPING, booleanMapping); } + @Test + public void parseDateField() { + NameValue faxConfirmed3 = new NameValue("m.faxConfirmed3", "2020-10-10"); + GeneratorPayload generatorPayload = new GeneratorPayload(faxConfirmed3); + ExportFacadePayload exportFacadePayload = new ExportFacadePayload("", "", generatorPayload, ""); + TypeParser typeParser = TypeParserFactory.of(exportFacadePayload); + typeParser.setPropertyInfo(new BaseGenerator.PropertyInfo(MedicalRecord.class, true)); + Object booleanMapping = typeParser.parse("faxConfirmed3"); + assertEquals(DATE_MAPPING, booleanMapping); + } + } \ No newline at end of file From c794665476befa3582ee7008ea2521c7374abf4b Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 5 Jan 2022 10:11:12 +0400 Subject: [PATCH 290/405] DDP-7121 | make general parser for dynamic fields --- .../model/elastic/export/ExportFacade.java | 7 +++- .../export/generate/MappingGenerator.java | 2 + .../export/generate/SourceGenerator.java | 2 + .../elastic/export/parse/BaseParser.java | 3 +- .../export/parse/BaseParserFactory.java | 30 +++++++++++++ .../parse/DynamicFieldsParser.java} | 42 +++++++++++++++---- .../model/elastic/export/parse/Parser.java | 1 + .../elastic/export/parse/TypeParser.java | 4 -- .../export/parse/TypeParserFactory.java | 22 ++-------- .../export/parse/ValueParserFactory.java | 12 ++++++ .../DynamicFieldsMappingMigrator.java | 5 ++- .../elastic/export/parse/TypeParserTest.java | 10 +++-- ...Test.java => DynamicFieldsParserTest.java} | 12 +++--- 13 files changed, 106 insertions(+), 46 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserFactory.java rename src/main/java/org/broadinstitute/dsm/model/elastic/{migration/DynamicFieldsTypeParser.java => export/parse/DynamicFieldsParser.java} (70%) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/ValueParserFactory.java rename src/test/java/org/broadinstitute/dsm/model/elastic/migration/{DynamicFieldsTypeParserTest.java => DynamicFieldsParserTest.java} (56%) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index 7539ecbd3..ddad579ed 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -12,9 +12,11 @@ import org.broadinstitute.dsm.model.elastic.export.generate.MappingGenerator; import org.broadinstitute.dsm.model.elastic.export.generate.MappingGeneratorFactory; import org.broadinstitute.dsm.model.elastic.export.generate.SourceGeneratorFactory; +import org.broadinstitute.dsm.model.elastic.export.parse.BaseParser; import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; import org.broadinstitute.dsm.model.elastic.export.parse.TypeParserFactory; import org.broadinstitute.dsm.model.elastic.export.parse.ValueParser; +import org.broadinstitute.dsm.model.elastic.export.parse.ValueParserFactory; import org.broadinstitute.dsm.model.elastic.export.process.BaseProcessor; import org.broadinstitute.dsm.model.elastic.export.process.ProcessorFactory; import org.broadinstitute.dsm.model.elastic.export.process.ProcessorFactoryImpl; @@ -52,7 +54,7 @@ private void upsertMapping() { String fieldName = exportFacadePayload.getCamelCaseFieldName(); propertyInfo.setFieldName(fieldName); generator = generatorFactory.make(propertyInfo); - TypeParser typeParser = TypeParserFactory.of(exportFacadePayload); + BaseParser typeParser = new TypeParserFactory().of(exportFacadePayload); typeParser.setPropertyInfo(propertyInfo); generator.setParser(typeParser); generator.setPayload(exportFacadePayload.getGeneratorPayload()); @@ -76,7 +78,8 @@ private ESDsm fetchData() { private Map processData(ESDsm esDsm) { BaseGenerator.PropertyInfo propertyInfo = getPropertyInfo(); - ValueParser valueParser = new ValueParser(); + BaseParser valueParser = new ValueParserFactory().of(exportFacadePayload); + valueParser.setPropertyInfo(propertyInfo); GeneratorFactory sourceGeneratorFactory = new SourceGeneratorFactory(); generator = sourceGeneratorFactory.make(propertyInfo); generator.setParser(valueParser); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java index d1cfb221a..890861d00 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java @@ -31,6 +31,7 @@ public Map generate() { @Override protected Object parseElement() { + parser.setFieldName(getFieldName()); return parser.parse(getFieldName()); } @@ -47,6 +48,7 @@ protected Map parseJson() { Map resultMap = new HashMap<>(); Map fieldsByValues = parseJsonToMapFromValue(); for (Map.Entry entry: fieldsByValues.entrySet()) { + parser.setFieldName(entry.getKey()); Object eachType = parser.parse(entry.getKey()); resultMap.put(Util.underscoresToCamelCase(entry.getKey()), eachType); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java index fbcaaba8b..0de8a7384 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java @@ -32,6 +32,7 @@ private Map buildPropertyLevelWithData(Object dataToExport) { @Override protected Object parseElement() { + parser.setFieldName(getFieldName()); return parser.parse(String.valueOf(getNameValue().getValue())); } @@ -45,6 +46,7 @@ protected Map parseJsonValuesToObject() { Map dynamicFieldValues = parseJsonToMapFromValue(); Map transformedMap = new HashMap<>(); for (Map.Entry entry : dynamicFieldValues.entrySet()) { + parser.setFieldName(entry.getKey()); transformedMap.put(Util.underscoresToCamelCase(entry.getKey()), parser.parse(String.valueOf(entry.getValue()))); } return transformedMap; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java index 3d72009d9..6fc887097 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java @@ -12,6 +12,7 @@ public abstract class BaseParser implements Parser { + protected static final String TYPE = "type"; protected BaseGenerator.PropertyInfo propertyInfo; protected String fieldName; protected String realm; @@ -33,7 +34,7 @@ public Object parse(String element) { Class propertyClass = propertyInfo.getPropertyClass(); Object elementMap; try { - Field field = propertyClass.getDeclaredField(element); + Field field = propertyClass.getDeclaredField(fieldName); if (long.class.isAssignableFrom(field.getType())) { elementMap = forNumeric(element); } else if (boolean.class.isAssignableFrom(field.getType())) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserFactory.java new file mode 100644 index 000000000..d39331e95 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParserFactory.java @@ -0,0 +1,30 @@ +package org.broadinstitute.dsm.model.elastic.export.parse; + +import org.broadinstitute.dsm.model.elastic.export.ExportFacadePayload; +import org.broadinstitute.dsm.statics.ESObjectConstants; + +public abstract class BaseParserFactory { + + public BaseParser of(ExportFacadePayload exportFacadePayload) { + BaseParser typeParser = getInitialParser(); + if (isDynamicFields(exportFacadePayload.getCamelCaseFieldName())) { + typeParser = buildDynamicFieldsParser(exportFacadePayload); + } + return typeParser; + } + + protected abstract BaseParser getInitialParser(); + + protected BaseParser buildDynamicFieldsParser(ExportFacadePayload exportFacadePayload) { + BaseParser typeParser; + DynamicFieldsParser dynamicFieldsParser = new DynamicFieldsParser(); + dynamicFieldsParser.setParser(getInitialParser()); + dynamicFieldsParser.setRealm(exportFacadePayload.getRealm()); + typeParser = dynamicFieldsParser; + return typeParser; + } + + protected boolean isDynamicFields(String fieldName) { + return ESObjectConstants.ADDITIONAL_VALUES_JSON.equals(fieldName) || ESObjectConstants.DATA.equals(fieldName); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/DynamicFieldsParser.java similarity index 70% rename from src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParser.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/DynamicFieldsParser.java index df760b559..71f0cbd67 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/DynamicFieldsParser.java @@ -1,9 +1,8 @@ -package org.broadinstitute.dsm.model.elastic.migration; +package org.broadinstitute.dsm.model.elastic.export.parse; import com.fasterxml.jackson.core.type.TypeReference; import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.dao.settings.FieldSettingsDao; -import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; import org.broadinstitute.dsm.util.ObjectMapperSingleton; import java.io.IOException; @@ -13,7 +12,7 @@ import static org.broadinstitute.dsm.model.Filter.NUMBER; -public class DynamicFieldsTypeParser extends TypeParser { +public class DynamicFieldsParser extends BaseParser { public static final String DATE_TYPE = "DATE"; public static final String CHECKBOX_TYPE = "CHECKBOX"; @@ -21,6 +20,7 @@ public class DynamicFieldsTypeParser extends TypeParser { public static final String ACTIVITY_TYPE = "ACTIVITY"; private String displayType; private String possibleValuesJson; + private BaseParser parser; public void setDisplayType(String displayType) { this.displayType = displayType; @@ -30,27 +30,31 @@ public void setPossibleValuesJson(String possibleValuesJson) { this.possibleValuesJson = possibleValuesJson; } + public void setParser(BaseParser parser) { + this.parser = parser; + } + @Override public Object parse(String fieldName) { if (StringUtils.isBlank(displayType)) - displayType = FieldSettingsDao.of().getDisplayTypeByInstanceNameAndColumnName(realm, fieldName).orElse(StringUtils.EMPTY); + displayType = FieldSettingsDao.of().getDisplayTypeByInstanceNameAndColumnName(realm, super.fieldName).orElse(StringUtils.EMPTY); Object parsedValue; if (DATE_TYPE.equals(displayType)) { - parsedValue = forDate(displayType); + parsedValue = forDate(fieldName); } else if (CHECKBOX_TYPE.equals(displayType)) { - parsedValue = forBoolean(displayType); + parsedValue = forBoolean(fieldName); } else if (isActivityRelatedType()) { Optional maybeType = getTypeFromPossibleValuesJson(); this.displayType = maybeType.orElse(StringUtils.EMPTY); parsedValue = maybeType .map(this::parse) - .orElse(forString(displayType)); + .orElse(forString(fieldName)); } else if (NUMBER.equals(displayType)) { - parsedValue = forNumeric(displayType); + parsedValue = forNumeric(fieldName); } else { - parsedValue = forString(displayType); + parsedValue = forString(fieldName); } displayType = null; @@ -58,6 +62,26 @@ public Object parse(String fieldName) { return parsedValue; } + @Override + protected Object forNumeric(String value) { + return parser.forNumeric(value); + } + + @Override + protected Object forBoolean(String value) { + return parser.forBoolean(value); + } + + @Override + protected Object forDate(String value) { + return parser.forDate(value); + } + + @Override + protected Object forString(String value) { + return parser.forString(value); + } + private Optional getTypeFromPossibleValuesJson() { try { List> possibleValues = ObjectMapperSingleton.instance().readValue(possibleValuesJson, new TypeReference>>() {}); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/Parser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/Parser.java index 9565f2687..4609ed665 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/Parser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/Parser.java @@ -6,4 +6,5 @@ public interface Parser { default Object[] parse(String[] values) { throw new UnsupportedOperationException(); } + void setFieldName(String fieldName); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java index cee297e01..6dfc5252c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParser.java @@ -1,19 +1,15 @@ package org.broadinstitute.dsm.model.elastic.export.parse; -import java.lang.reflect.Field; import java.util.HashMap; import java.util.Map; -import org.broadinstitute.dsm.db.structure.DbDateConversion; import org.broadinstitute.dsm.model.elastic.export.generate.MappingGenerator; -import org.elasticsearch.common.text.Text; public class TypeParser extends BaseParser { private static final String TEXT = "text"; private static final String FIELDS = "fields"; private static final String KEYWORD = "keyword"; - protected static final String TYPE = "type"; public static final Map TEXT_KEYWORD_MAPPING = new HashMap<>( new HashMap<>( Map.of(TYPE, TEXT, diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParserFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParserFactory.java index f7d020c25..bad24e905 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParserFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParserFactory.java @@ -1,25 +1,11 @@ package org.broadinstitute.dsm.model.elastic.export.parse; -import org.broadinstitute.dsm.db.dao.settings.FieldSettingsDao; -import org.broadinstitute.dsm.model.elastic.export.ExportFacadePayload; -import org.broadinstitute.dsm.model.elastic.migration.DynamicFieldsTypeParser; -import org.broadinstitute.dsm.statics.ESObjectConstants; +public class TypeParserFactory extends BaseParserFactory { -import java.util.Optional; + private static final TypeParser typeParser = new TypeParser(); -public class TypeParserFactory { - - public static TypeParser of(ExportFacadePayload exportFacadePayload) { - TypeParser typeParser = new TypeParser(); - if (isDynamicFields(exportFacadePayload.getCamelCaseFieldName())) { - DynamicFieldsTypeParser dynamicFieldsTypeParser = new DynamicFieldsTypeParser(); - dynamicFieldsTypeParser.setRealm(exportFacadePayload.getRealm()); - typeParser = dynamicFieldsTypeParser; - } + @Override + protected BaseParser getInitialParser() { return typeParser; } - - private static boolean isDynamicFields(String fieldName) { - return ESObjectConstants.ADDITIONAL_VALUES_JSON.equals(fieldName) || ESObjectConstants.DATA.equals(fieldName); - } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/ValueParserFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/ValueParserFactory.java new file mode 100644 index 000000000..4929255a1 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/ValueParserFactory.java @@ -0,0 +1,12 @@ +package org.broadinstitute.dsm.model.elastic.export.parse; + +public class ValueParserFactory extends BaseParserFactory { + + private static final ValueParser valueParser = new ValueParser(); + + @Override + protected BaseParser getInitialParser() { + return valueParser; + } + +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java index 9f707f963..e26341ef1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java @@ -9,6 +9,7 @@ import org.broadinstitute.dsm.model.elastic.export.RequestPayload; import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; import org.broadinstitute.dsm.model.elastic.export.generate.MappingGenerator; +import org.broadinstitute.dsm.model.elastic.export.parse.DynamicFieldsParser; import java.util.HashMap; import java.util.List; @@ -22,7 +23,7 @@ public class DynamicFieldsMappingMigrator implements Exportable { public static final String DYNAMIC_FIELDS_WRAPPER_NAME = "dynamicFields"; private final String index; private final String study; - public DynamicFieldsTypeParser parser; + public DynamicFieldsParser parser; public Map propertyMap; private ElasticMappingExportAdapter elasticMappingExportAdapter; @@ -30,7 +31,7 @@ public class DynamicFieldsMappingMigrator implements Exportable { public DynamicFieldsMappingMigrator(String index, String study) { this.index = index; this.study = study; - this.parser = new DynamicFieldsTypeParser(); + this.parser = new DynamicFieldsParser(); this.propertyMap = new HashMap<>(); elasticMappingExportAdapter = new ElasticMappingExportAdapter(); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParserTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParserTest.java index 754c3ec01..f23cbc596 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParserTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/parse/TypeParserTest.java @@ -9,7 +9,6 @@ import org.broadinstitute.dsm.model.elastic.export.ExportFacadePayload; import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; -import org.broadinstitute.dsm.model.elastic.migration.DynamicFieldsTypeParser; import org.junit.Assert; import org.junit.Test; @@ -22,7 +21,8 @@ public void parseDynamicFields() { BaseGenerator.PropertyInfo propertyInfo = new BaseGenerator.PropertyInfo(MedicalRecord.class, true); propertyInfo.setFieldName("additionalValuesJson"); - DynamicFieldsTypeParser typeParser = new DynamicFieldsTypeParser(); + DynamicFieldsParser typeParser = new DynamicFieldsParser(); + typeParser.setParser(new TypeParser()); typeParser.setPropertyInfo(propertyInfo); typeParser.setFieldName("scooby"); typeParser.setDisplayType("TEXT"); @@ -47,7 +47,8 @@ public void parseNonDynamicFields() { NameValue crRequired = new NameValue("m.crRequired", true); GeneratorPayload generatorPayload = new GeneratorPayload(crRequired); ExportFacadePayload exportFacadePayload = new ExportFacadePayload("", "", generatorPayload, ""); - TypeParser typeParser = TypeParserFactory.of(exportFacadePayload); + BaseParser typeParser = new TypeParserFactory().of(exportFacadePayload); + typeParser.setFieldName("crRequired"); typeParser.setPropertyInfo(new BaseGenerator.PropertyInfo(MedicalRecord.class, true)); Object booleanMapping = typeParser.parse("crRequired"); assertEquals(BOOLEAN_MAPPING, booleanMapping); @@ -58,7 +59,8 @@ public void parseDateField() { NameValue faxConfirmed3 = new NameValue("m.faxConfirmed3", "2020-10-10"); GeneratorPayload generatorPayload = new GeneratorPayload(faxConfirmed3); ExportFacadePayload exportFacadePayload = new ExportFacadePayload("", "", generatorPayload, ""); - TypeParser typeParser = TypeParserFactory.of(exportFacadePayload); + BaseParser typeParser = new TypeParserFactory().of(exportFacadePayload); + typeParser.setFieldName("faxConfirmed3"); typeParser.setPropertyInfo(new BaseGenerator.PropertyInfo(MedicalRecord.class, true)); Object booleanMapping = typeParser.parse("faxConfirmed3"); assertEquals(DATE_MAPPING, booleanMapping); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParserTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsParserTest.java similarity index 56% rename from src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParserTest.java rename to src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsParserTest.java index f1fa7bb5f..39bc6572f 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsTypeParserTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsParserTest.java @@ -4,21 +4,21 @@ import java.util.Map; -import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; import org.broadinstitute.dsm.model.elastic.export.generate.MappingGenerator; +import org.broadinstitute.dsm.model.elastic.export.parse.DynamicFieldsParser; import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; import org.junit.Test; -public class DynamicFieldsTypeParserTest { +public class DynamicFieldsParserTest { @Test public void parse() { String possibleValuesJson = "[{\"value\":\"CONSENT.completedAt\",\"type\":\"DATE\"}]"; String displayType = "ACTIVITY_STAFF"; - DynamicFieldsTypeParser dynamicFieldsTypeParser = new DynamicFieldsTypeParser(); - dynamicFieldsTypeParser.setDisplayType(displayType); - dynamicFieldsTypeParser.setPossibleValuesJson(possibleValuesJson); - Map mapping = (Map)dynamicFieldsTypeParser.parse(displayType); + DynamicFieldsParser dynamicFieldsParser = new DynamicFieldsParser(); + dynamicFieldsParser.setDisplayType(displayType); + dynamicFieldsParser.setPossibleValuesJson(possibleValuesJson); + Map mapping = (Map) dynamicFieldsParser.parse(displayType); Object date = mapping.get(MappingGenerator.TYPE); assertEquals(TypeParser.DATE, date); } From 8411a637c309d81979dc4a09d15b64bd8b98c8bb Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 5 Jan 2022 12:30:22 +0400 Subject: [PATCH 291/405] DDP-7121 | clean up the code, remove unused methods and comments, fix bugs --- .../dsm/db/dao/settings/FieldSettingsDao.java | 7 ++++- .../dsm/model/elastic/Util.java | 13 ++++++-- .../elastic/export/parse/BaseParser.java | 31 ------------------- .../export/parse/DynamicFieldsParser.java | 22 ++++++++----- .../dsm/model/elastic/UtilTest.java | 31 ++++++++++++++++--- 5 files changed, 57 insertions(+), 47 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/dao/settings/FieldSettingsDao.java b/src/main/java/org/broadinstitute/dsm/db/dao/settings/FieldSettingsDao.java index ef1ffb652..817027eb0 100644 --- a/src/main/java/org/broadinstitute/dsm/db/dao/settings/FieldSettingsDao.java +++ b/src/main/java/org/broadinstitute/dsm/db/dao/settings/FieldSettingsDao.java @@ -90,7 +90,8 @@ public class FieldSettingsDao implements Dao { private static final String LAST_CHANGED = "last_changed"; private static final String CHANGED_BY = "changed_by"; - private FieldSettingsDao() {} + // for test purposes only + protected FieldSettingsDao() {} public static FieldSettingsDao of() { if (fieldSettingsDao == null) { @@ -99,6 +100,10 @@ public static FieldSettingsDao of() { return fieldSettingsDao; } + public static void setInstance(FieldSettingsDao fieldSettingsDao) { + FieldSettingsDao.fieldSettingsDao = fieldSettingsDao; + } + @Override public int create(FieldSettingsDto fieldSettingsDto) { SimpleResult simpleResult = inTransaction(conn -> { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index fde95cf1b..388366c89 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -17,6 +17,9 @@ import org.broadinstitute.dsm.db.structure.ColumnName; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; +import org.broadinstitute.dsm.model.elastic.export.parse.DynamicFieldsParser; +import org.broadinstitute.dsm.model.elastic.export.parse.ValueParser; +import org.broadinstitute.dsm.model.elastic.export.parse.ValueParserFactory; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.statics.ESObjectConstants; @@ -42,6 +45,12 @@ public class Util { private static final Pattern CAMEL_CASE_REGEX = Pattern.compile("(([a-z])+([A-z])+(\\.)*)*"); private static final Pattern UPPER_CASE_REGEX = Pattern.compile("(?=\\p{Upper})"); public static final Gson GSON = new Gson(); + public static final DynamicFieldsParser DYNAMIC_FIELDS_PARSER = new DynamicFieldsParser(); + public static final ValueParser PARSER = new ValueParser(); + + static { + DYNAMIC_FIELDS_PARSER.setParser(PARSER); + } public static String getQueryTypeFromId(String id) { String type; @@ -129,8 +138,8 @@ static Map convertToMap(String fieldName, Object fieldValue) { Map transformedMap = new HashMap<>(); for (Map.Entry object: objectMap.entrySet()) { String field = object.getKey(); - Object value = object.getValue(); - if (FamilyMemberConstants.IS_APPLICANT.equals(field)) value = Boolean.valueOf(String.valueOf(value)); + DYNAMIC_FIELDS_PARSER.setFieldName(field); + Object value = DYNAMIC_FIELDS_PARSER.parse(String.valueOf(object.getValue())); String camelCaseField = underscoresToCamelCase(field); transformedMap.put(camelCaseField, value); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java index 6fc887097..673f5e94d 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/BaseParser.java @@ -61,37 +61,6 @@ public Object parse(String element) { protected abstract Object forString(String value); - boolean isDateOrTimeOrDateTime(String value) { - return isDate(value) || isTime(value) || isDateTime(value); - } - - boolean isDateTime(String value) { - try { - LocalDateTime.parse(value); - } catch (DateTimeParseException dtpe) { - return false; - } - return true; - } - - boolean isTime(String value) { - try { - LocalTime.parse(value); - } catch (DateTimeParseException dtpe) { - return false; - } - return true; - } - - boolean isDate(String value) { - try { - LocalDate.parse(value); - } catch (DateTimeParseException dtpe) { - return false; - } - return true; - } - protected boolean isBoolean(String value) { return convertBoolean(value).equalsIgnoreCase(Boolean.TRUE.toString()) || convertBoolean(value).equalsIgnoreCase(Boolean.FALSE.toString()); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/DynamicFieldsParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/DynamicFieldsParser.java index 71f0cbd67..9f18632c8 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/DynamicFieldsParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/DynamicFieldsParser.java @@ -21,6 +21,7 @@ public class DynamicFieldsParser extends BaseParser { private String displayType; private String possibleValuesJson; private BaseParser parser; + public FieldSettingsDao fieldSettingsDao = FieldSettingsDao.of(); public void setDisplayType(String displayType) { this.displayType = displayType; @@ -34,27 +35,32 @@ public void setParser(BaseParser parser) { this.parser = parser; } + public void setFieldSettingsDao(FieldSettingsDao fieldSettingsDao) { + this.fieldSettingsDao = fieldSettingsDao; + } + @Override - public Object parse(String fieldName) { + public Object parse(String element) { - if (StringUtils.isBlank(displayType)) - displayType = FieldSettingsDao.of().getDisplayTypeByInstanceNameAndColumnName(realm, super.fieldName).orElse(StringUtils.EMPTY); + if (StringUtils.isBlank(displayType)) { + displayType = fieldSettingsDao.getDisplayTypeByInstanceNameAndColumnName(realm, super.fieldName).orElse(StringUtils.EMPTY); + } Object parsedValue; if (DATE_TYPE.equals(displayType)) { - parsedValue = forDate(fieldName); + parsedValue = forDate(element); } else if (CHECKBOX_TYPE.equals(displayType)) { - parsedValue = forBoolean(fieldName); + parsedValue = forBoolean(element); } else if (isActivityRelatedType()) { Optional maybeType = getTypeFromPossibleValuesJson(); this.displayType = maybeType.orElse(StringUtils.EMPTY); parsedValue = maybeType .map(this::parse) - .orElse(forString(fieldName)); + .orElse(forString(element)); } else if (NUMBER.equals(displayType)) { - parsedValue = forNumeric(fieldName); + parsedValue = forNumeric(element); } else { - parsedValue = forString(fieldName); + parsedValue = forString(element); } displayType = null; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java index 12f59a0f6..e5170685a 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java @@ -4,16 +4,16 @@ import static org.junit.Assert.*; import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; +import java.util.*; +import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.MedicalRecord; import org.broadinstitute.dsm.db.OncHistoryDetail; import org.broadinstitute.dsm.db.Participant; import org.broadinstitute.dsm.db.Tissue; +import org.broadinstitute.dsm.db.dao.settings.FieldSettingsDao; import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; +import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; import org.broadinstitute.dsm.model.FollowUp; import org.junit.Assert; import org.junit.Test; @@ -58,7 +58,26 @@ public void transformObjectToMap() { @Test public void transformJsonToMap() { - String json = "{\"DDP_INSTANCE\": \"TEST\", \"DDP_VALUE\": \"VALUE\"}"; + + class FieldSettingsDaoMock extends FieldSettingsDao { + + @Override + public Optional getDisplayTypeByInstanceNameAndColumnName(String instanceName, String columnName) { + String displayType = StringUtils.EMPTY; + if ("BOOLEAN_VAL".equals(columnName)) { + displayType = "CHECKBOX"; + } else if ("LONG_VAL".equals(columnName)) { + displayType = "NUMBER"; + } + return Optional.of(displayType); + } + } + + FieldSettingsDaoMock fieldSettingsDaoMock = new FieldSettingsDaoMock(); + + FieldSettingsDao.setInstance(fieldSettingsDaoMock); + + String json = "{\"DDP_INSTANCE\": \"TEST\", \"DDP_VALUE\": \"VALUE\", \"BOOLEAN_VAL\": \"true\", \"LONG_VAL\": \"5\"}"; ParticipantDataDto participantDataDto = new ParticipantDataDto.Builder() .withParticipantDataId(10) @@ -71,6 +90,8 @@ public void transformJsonToMap() { Map result = Util.transformObjectToMap(participantDataDto); assertEquals("TEST", ((Map) result.get("dynamicFields")).get("ddpInstance")); assertEquals("VALUE", ((Map) result.get("dynamicFields")).get("ddpValue")); + assertEquals(true, ((Map) result.get("dynamicFields")).get("booleanVal")); + assertEquals(5L, ((Map) result.get("dynamicFields")).get("longVal")); } @Test From 03f154ad3de43642adf474fcec014c68a00af241 Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 5 Jan 2022 16:48:52 +0400 Subject: [PATCH 292/405] DDP-7121 | fix bugs for non-existing dynamic fields --- .../dsm/db/dao/settings/FieldSettingsDao.java | 2 +- .../broadinstitute/dsm/model/elastic/Util.java | 17 ++++++----------- .../export/parse/DynamicFieldsParser.java | 2 ++ .../migration/BaseCollectionMigrator.java | 2 +- .../elastic/migration/BaseSingleMigrator.java | 2 +- .../migration/DynamicFieldsMappingMigrator.java | 3 +++ .../dsm/model/elastic/UtilTest.java | 9 ++------- .../DynamicFieldsMappingMigratorTest.java | 4 ++-- 8 files changed, 18 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/dao/settings/FieldSettingsDao.java b/src/main/java/org/broadinstitute/dsm/db/dao/settings/FieldSettingsDao.java index 817027eb0..50256b0af 100644 --- a/src/main/java/org/broadinstitute/dsm/db/dao/settings/FieldSettingsDao.java +++ b/src/main/java/org/broadinstitute/dsm/db/dao/settings/FieldSettingsDao.java @@ -184,7 +184,7 @@ public Optional getDisplayTypeByInstanceNameAndColumnName(String instanc throw new RuntimeException("could not find the specified display type by instance name and column name", simpleResult.resultException); } - return Optional.of((String) simpleResult.resultValue); + return Optional.ofNullable((String) simpleResult.resultValue); } public List getOptionAndRadioFieldSettingsByInstanceId(int instanceId) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 388366c89..285508ae5 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -8,9 +8,6 @@ import java.util.stream.Collectors; import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.api.client.json.Json; -import com.google.common.reflect.TypeToken; import com.google.gson.Gson; import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.*; @@ -19,10 +16,7 @@ import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; import org.broadinstitute.dsm.model.elastic.export.parse.DynamicFieldsParser; import org.broadinstitute.dsm.model.elastic.export.parse.ValueParser; -import org.broadinstitute.dsm.model.elastic.export.parse.ValueParserFactory; -import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import org.broadinstitute.dsm.statics.DBConstants; -import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ObjectMapperSingleton; import org.broadinstitute.dsm.util.ParticipantUtil; import org.broadinstitute.dsm.util.PatchUtil; @@ -93,15 +87,15 @@ private static boolean hasNoUnderscores(String[] splittedWords) { return splittedWords.length < 2; } - public static List> transformObjectCollectionToCollectionMap(List values) { + public static List> transformObjectCollectionToCollectionMap(List values, String realm) { List> result = new ArrayList<>(); for (Object obj : values) { - result.add(transformObjectToMap(obj)); + result.add(transformObjectToMap(obj, realm)); } return result; } - public static Map transformObjectToMap(Object obj) { + public static Map transformObjectToMap(Object obj, String realm) { Map map = new HashMap<>(); List declaredFields = new ArrayList<>(List.of(obj.getClass().getDeclaredFields())); List declaredFieldsSuper = new ArrayList<>(List.of(obj.getClass().getSuperclass().getDeclaredFields())); @@ -117,7 +111,7 @@ public static Map transformObjectToMap(Object obj) { if (Objects.isNull(fieldValue)) { continue; } - map.putAll(convertToMap(annotation.value(), fieldValue)); + map.putAll(convertToMap(annotation.value(), fieldValue, realm)); } catch (IllegalAccessException e) { throw new RuntimeException(e); } @@ -125,7 +119,7 @@ public static Map transformObjectToMap(Object obj) { return map; } - static Map convertToMap(String fieldName, Object fieldValue) { + static Map convertToMap(String fieldName, Object fieldValue, String realm) { Map finalResult; switch (fieldName) { case "follow_ups": @@ -139,6 +133,7 @@ static Map convertToMap(String fieldName, Object fieldValue) { for (Map.Entry object: objectMap.entrySet()) { String field = object.getKey(); DYNAMIC_FIELDS_PARSER.setFieldName(field); + DYNAMIC_FIELDS_PARSER.setRealm(realm); Object value = DYNAMIC_FIELDS_PARSER.parse(String.valueOf(object.getValue())); String camelCaseField = underscoresToCamelCase(field); transformedMap.put(camelCaseField, value); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/DynamicFieldsParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/DynamicFieldsParser.java index 9f18632c8..615864a94 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/DynamicFieldsParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/DynamicFieldsParser.java @@ -39,6 +39,8 @@ public void setFieldSettingsDao(FieldSettingsDao fieldSettingsDao) { this.fieldSettingsDao = fieldSettingsDao; } + + @Override public Object parse(String element) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java index 2c577c5ae..39a453fbd 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigrator.java @@ -26,6 +26,6 @@ public Map generate() { @Override protected void transformObject(Object object) { - transformedList = Util.transformObjectCollectionToCollectionMap((List) object); + transformedList = Util.transformObjectCollectionToCollectionMap((List) object, realm); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseSingleMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseSingleMigrator.java index 629c5aa5f..a64886a4a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseSingleMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseSingleMigrator.java @@ -21,6 +21,6 @@ public Map generate() { @Override protected void transformObject(Object object) { - transformedObject = Util.transformObjectToMap(object); + transformedObject = Util.transformObjectToMap(object, realm); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java index e26341ef1..cc6a88a75 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigrator.java @@ -10,6 +10,8 @@ import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; import org.broadinstitute.dsm.model.elastic.export.generate.MappingGenerator; import org.broadinstitute.dsm.model.elastic.export.parse.DynamicFieldsParser; +import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; +import org.broadinstitute.dsm.model.elastic.export.parse.TypeParserFactory; import java.util.HashMap; import java.util.List; @@ -32,6 +34,7 @@ public DynamicFieldsMappingMigrator(String index, String study) { this.index = index; this.study = study; this.parser = new DynamicFieldsParser(); + this.parser.setParser(new TypeParser()); this.propertyMap = new HashMap<>(); elasticMappingExportAdapter = new ElasticMappingExportAdapter(); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java index e5170685a..eef0cbde3 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java @@ -1,19 +1,14 @@ package org.broadinstitute.dsm.model.elastic; -import static org.broadinstitute.dsm.model.participant.data.ParticipantData.GSON; import static org.junit.Assert.*; import java.lang.reflect.Field; import java.util.*; import org.apache.commons.lang3.StringUtils; -import org.broadinstitute.dsm.db.MedicalRecord; -import org.broadinstitute.dsm.db.OncHistoryDetail; import org.broadinstitute.dsm.db.Participant; -import org.broadinstitute.dsm.db.Tissue; import org.broadinstitute.dsm.db.dao.settings.FieldSettingsDao; import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; -import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; import org.broadinstitute.dsm.model.FollowUp; import org.junit.Assert; import org.junit.Test; @@ -48,7 +43,7 @@ public void transformObjectToMap() { "2020-10-28", "2020-10-28", "2020-10-28", "ptNotes", true, true, "additionalValuesJson", 1934283746283L); - Map transformedObject = Util.transformObjectToMap(participant); + Map transformedObject = Util.transformObjectToMap(participant, "angio"); assertEquals("1", transformedObject.get("participantId")); assertEquals("QWERTY", transformedObject.get("ddpParticipantId")); assertEquals("2020-10-28", transformedObject.get("created")); @@ -87,7 +82,7 @@ public Optional getDisplayTypeByInstanceNameAndColumnName(String instanc .withData(json) .build(); - Map result = Util.transformObjectToMap(participantDataDto); + Map result = Util.transformObjectToMap(participantDataDto, "angio"); assertEquals("TEST", ((Map) result.get("dynamicFields")).get("ddpInstance")); assertEquals("VALUE", ((Map) result.get("dynamicFields")).get("ddpValue")); assertEquals(true, ((Map) result.get("dynamicFields")).get("booleanVal")); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java index ccd9566b3..9e6ccf8f0 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java @@ -21,12 +21,12 @@ public void testExport() { final String study = "angio"; List exportables = Arrays.asList( //DynamicFieldsMappingMigrator should be first in the list to make sure that mapping will be exported for first + new ParticipantDataMigrator(index, study), + new ParticipantMigrator(index, study), new DynamicFieldsMappingMigrator(index, study), new OncHistoryMigrator(index, study), new MedicalRecordMigrator(index, study), new OncHistoryDetailsMigrator(index, study), - new ParticipantDataMigrator(index, study), - new ParticipantMigrator(index, study), new KitRequestShippingMigrator(index, study), new TissueMigrator(index, study)); exportables.forEach(Exportable::export); From 4309406cc36f55d030c13f873d7b77467475b18a Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 5 Jan 2022 17:09:51 +0400 Subject: [PATCH 293/405] DDP-7121 | change the json parsing mechanism from Gson to Jackson --- .../dsm/model/elastic/export/generate/BaseGenerator.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java index d66f575c0..d26e2e5cd 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java @@ -1,5 +1,6 @@ package org.broadinstitute.dsm.model.elastic.export.generate; +import java.io.IOException; import java.util.Map; import java.util.Objects; @@ -11,6 +12,7 @@ import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; +import org.broadinstitute.dsm.util.ObjectMapperSingleton; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -89,7 +91,11 @@ public Object collect() { protected abstract T parseJson(); protected Map parseJsonToMapFromValue() { - return GSON.fromJson(String.valueOf(getNameValue().getValue()), Map.class); + try { + return ObjectMapperSingleton.instance().readValue(String.valueOf(getNameValue().getValue()), Map.class); + } catch (IOException e) { + throw new RuntimeException(e); + } } protected Object parseSingleElement() { From d06c130b89e6e8b5833570bf267dc7eaaba0a0c5 Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 6 Jan 2022 12:35:01 +0400 Subject: [PATCH 294/405] DDP-7121 | fix bugs related to json parsing exceptions and NOT operator --- .../org/broadinstitute/dsm/db/MedicalRecord.java | 8 ++++---- .../model/elastic/export/generate/BaseGenerator.java | 6 ++++-- .../elastic/export/generate/JsonParseException.java | 8 ++++++++ .../elastic/export/generate/MappingGenerator.java | 1 + .../elastic/export/generate/SourceGenerator.java | 1 + .../model/elastic/filter/AndOrFilterSeparator.java | 2 +- .../dsm/model/elastic/filter/Operator.java | 7 +++++-- .../model/elastic/filter/QueryBuilderFactory.java | 1 + .../org/broadinstitute/dsm/route/DashboardRoute.java | 2 +- .../elastic/filter/AndOrFilterSeparatorTest.java | 12 ++++++++++++ .../dsm/model/elastic/filter/OperatorTest.java | 5 +++++ 11 files changed, 43 insertions(+), 10 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/JsonParseException.java diff --git a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java index 20bab003d..6a3a49177 100644 --- a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java +++ b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java @@ -276,11 +276,11 @@ public class MedicalRecord { primaryKey = DBConstants.MEDICAL_RECORD_ID, columnPrefix = "") @ColumnName (DBConstants.FOLLOWUP_REQUIRED) - private boolean followUpRequired; + private boolean followupRequired; @JsonProperty("followupRequired") - public boolean isFollowUpRequired() { - return followUpRequired; + public boolean isFollowupRequired() { + return followupRequired; } @TableName ( @@ -412,7 +412,7 @@ public MedicalRecord(long medicalRecordId, long institutionId, String ddpInstitu this.notes = mrNotes; this.reviewMedicalRecord = reviewMedicalRecord; this.followUps = followUps; - this.followUpRequired = followUpRequired; + this.followupRequired = followUpRequired; this.followUpRequiredText = followUpRequiredText; this.additionalValuesJson = additionalValuesJson; this.unableObtainText = unableObtainText; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java index d26e2e5cd..95dc7b35c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java @@ -4,8 +4,8 @@ import java.util.Map; import java.util.Objects; +import com.fasterxml.jackson.databind.JsonMappingException; import com.google.gson.Gson; -import com.google.gson.JsonSyntaxException; import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.db.structure.TableName; @@ -82,7 +82,7 @@ public Object collect() { Object sourceToUpsert; try { sourceToUpsert = parseJson(); - } catch (JsonSyntaxException jse) { + } catch (JsonParseException jpe) { sourceToUpsert = parseSingleElement(); } return sourceToUpsert; @@ -93,6 +93,8 @@ public Object collect() { protected Map parseJsonToMapFromValue() { try { return ObjectMapperSingleton.instance().readValue(String.valueOf(getNameValue().getValue()), Map.class); + } catch (com.fasterxml.jackson.core.JsonParseException | JsonMappingException je) { + throw new JsonParseException(je.getMessage()); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/JsonParseException.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/JsonParseException.java new file mode 100644 index 000000000..ca94e10bc --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/JsonParseException.java @@ -0,0 +1,8 @@ +package org.broadinstitute.dsm.model.elastic.export.generate; + +public class JsonParseException extends RuntimeException { + + public JsonParseException(String message) { + super(message); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java index 890861d00..bc2da1702 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGenerator.java @@ -3,6 +3,7 @@ import java.util.HashMap; import java.util.Map; +import com.fasterxml.jackson.core.JsonParseException; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import org.broadinstitute.dsm.statics.ESObjectConstants; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java index 0de8a7384..313786399 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGenerator.java @@ -3,6 +3,7 @@ import java.util.HashMap; import java.util.Map; +import com.fasterxml.jackson.core.JsonParseException; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import org.broadinstitute.dsm.statics.ESObjectConstants; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java index 44149b964..be2a8154c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java @@ -10,7 +10,7 @@ public class AndOrFilterSeparator { - public static final String DSM_ALIAS_REGEX = "(m|p|r|t|d|o|k|JS|ST|DA|\\()(\\.|\\s)*([a-z]|O|R|T|D|)(\\.)*"; + public static final String DSM_ALIAS_REGEX = "(NO|m|p|r|t|d|o|k|JS|ST|DA|\\()(\\.|\\s)*([a-z]|O|R|T|D|)(\\.)*"; public static final String OR_DSM_ALIAS_REGEX = "(OR) " + DSM_ALIAS_REGEX; public static final String AND_DSM_ALIAS_REGEX = "(AND) " + DSM_ALIAS_REGEX; public static final int AND_PATTERN_MATCHER_NUMBER = 7; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java index 6e7b85cf5..c64423f42 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java @@ -51,8 +51,11 @@ else if (filter.contains(Filter.JSON_EXTRACT)) return JSON_EXTRACT; String operator = Arrays.stream(filter.split(" ")) .filter(StringUtils::isNotBlank) - .collect(Collectors.toList()) - .get(1); + .filter(str -> Arrays.stream(Operator.values()).anyMatch(op -> op.value.equals(str))) + .findFirst() + .orElse(StringUtils.EMPTY); + // [NOT , m.mr_problem, <=>, 1] + // return getOperator(operator); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java index af94a77bb..311293f51 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java @@ -14,6 +14,7 @@ public static QueryBuilder buildQueryBuilder(Operator operator, QueryPayload pay case LIKE: case EQUALS: case DATE: + case DIAMOND_EQUALS: case STR_DATE: qb = new MatchQueryBuilder(payload.getFieldName(), payload.getValues()[0]); break; diff --git a/src/main/java/org/broadinstitute/dsm/route/DashboardRoute.java b/src/main/java/org/broadinstitute/dsm/route/DashboardRoute.java index eea895d22..3f7a3ab42 100644 --- a/src/main/java/org/broadinstitute/dsm/route/DashboardRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/DashboardRoute.java @@ -359,7 +359,7 @@ private void countMedicalRecordData(@NonNull List medicalRecordLi incrementCounter(dashboardValuesDetailed, "unableToObtainMedicalRecord"); foundAtPT.add("unableToObtainMedicalRecord"); } - if (medicalRecord.isFollowUpRequired()) { + if (medicalRecord.isFollowupRequired()) { incrementCounter(dashboardValuesDetailed, "followupRequiredMedicalRecord"); foundAtPT.add("followupRequiredMedicalRecord"); if (medicalRecord.getFollowUps() == null || medicalRecord.getFollowUps().length == 0) { diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparatorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparatorTest.java index 0a2981eaf..2d884f836 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparatorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparatorTest.java @@ -56,6 +56,18 @@ public void parseFiltersByLogicalOperatorsSingle() { } + @Test + public void parseFiltersByLogicalOperatorsSingle2() { + + String filter = " AND NOT m.mr_problem <=> 1 "; + Map> stringListMap = new AndOrFilterSeparator(filter).parseFiltersByLogicalOperators(); + Assert.assertEquals("NOT m.mr_problem <=> 1", stringListMap.get("AND").get(0)); + + + // + // + } + } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java index 1c86cc0c0..fee780afe 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java @@ -31,9 +31,14 @@ public void getOperator() { public void extractLogicalOperator() { String filterEquals = "m.medicalRecordId = 15"; String filterLike = "m.medicalRecordId LIKE 15"; + String notFilter = "NOT m.mrProblem <=> 1"; + Operator equalsOperator = Operator.extract(filterEquals); Operator likeOperator = Operator.extract(filterLike); + Operator diamondsOperator = Operator.extract(notFilter); + assertEquals(Operator.EQUALS, equalsOperator); assertEquals(Operator.LIKE, likeOperator); + assertEquals(Operator.DIAMOND_EQUALS, diamondsOperator); } } \ No newline at end of file From 88956c6832d6e28fa5325d5790a9d2cea82a1c1a Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 6 Jan 2022 16:07:09 +0400 Subject: [PATCH 295/405] DDP-7121 | typo fix --- .../java/org/broadinstitute/dsm/db/MedicalRecord.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java index 6a3a49177..066585067 100644 --- a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java +++ b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java @@ -289,10 +289,10 @@ public boolean isFollowupRequired() { primaryKey = DBConstants.MEDICAL_RECORD_ID, columnPrefix = "") @ColumnName (DBConstants.FOLLOWUP_REQUIRED_TEXT) - private String followUpRequiredText; + private String followupRequiredText; @JsonProperty("followupRequiredText") - public String getFollowUpRequiredText() { return followUpRequiredText; } + public String getFollowupRequiredText() { return followupRequiredText; } @TableName ( name = DBConstants.DDP_MEDICAL_RECORD, @@ -380,7 +380,7 @@ public MedicalRecord(long medicalRecordId, long institutionId, String ddpInstitu String mrReceived, String mrDocument, String mrDocumentFileNames, boolean mrProblem, String mrProblemText, boolean unableObtain, boolean duplicate, boolean international, boolean crRequired, String pathologyPresent, String mrNotes, boolean reviewMedicalRecord, - FollowUp[] followUps, boolean followUpRequired, String followUpRequiredText, String additionalValuesJson, + FollowUp[] followUps, boolean followUpRequired, String followupRequiredText, String additionalValuesJson, String unableObtainText, String ddpParticipantId) { this.medicalRecordId = medicalRecordId; this.institutionId = institutionId; @@ -413,7 +413,7 @@ public MedicalRecord(long medicalRecordId, long institutionId, String ddpInstitu this.reviewMedicalRecord = reviewMedicalRecord; this.followUps = followUps; this.followupRequired = followUpRequired; - this.followUpRequiredText = followUpRequiredText; + this.followupRequiredText = followupRequiredText; this.additionalValuesJson = additionalValuesJson; this.unableObtainText = unableObtainText; this.ddpParticipantId = ddpParticipantId; From 78b1fc5865bd4a19e3b05a740e1ab00c4ee7ab12 Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 6 Jan 2022 18:46:12 +0400 Subject: [PATCH 296/405] DDP-7121 | add json parsing as a static method to the ObjectMapperSingleton --- .../participant/BaseFilterParticipantList.java | 3 ++- .../dsm/util/ObjectMapperSingleton.java | 13 +++++++++++++ .../migration/DynamicFieldsMappingMigratorTest.java | 6 +++--- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java b/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java index e9be12ccd..a65896977 100644 --- a/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java +++ b/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java @@ -26,6 +26,7 @@ import org.broadinstitute.dsm.model.participant.ParticipantWrapperResult; import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; +import org.broadinstitute.dsm.util.ObjectMapperSingleton; import org.broadinstitute.dsm.util.ParticipantUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -158,7 +159,7 @@ public void addParticipantDataIdsForFilters(Filter filter, String fieldName, Lis continue; } String ddpParticipantId = participantData.getDdpParticipantId().orElse(null); - Map dataMap = GSON.fromJson(data, Map.class); + Map dataMap = ObjectMapperSingleton.readValue(data, Map.class); boolean questionWithOptions = (OPTIONS.equals(filter.getType()) || RADIO.equals(filter.getType())) && filter.getSelectedOptions() != null; boolean notEmptyCheck = filter.isNotEmpty() && dataMap.get(fieldName) != null && !dataMap.get(fieldName).isEmpty(); boolean emptyCheck = filter.isEmpty() && (dataMap.get(fieldName) == null || dataMap.get(fieldName).isEmpty()); diff --git a/src/main/java/org/broadinstitute/dsm/util/ObjectMapperSingleton.java b/src/main/java/org/broadinstitute/dsm/util/ObjectMapperSingleton.java index 3732f70cb..159963471 100644 --- a/src/main/java/org/broadinstitute/dsm/util/ObjectMapperSingleton.java +++ b/src/main/java/org/broadinstitute/dsm/util/ObjectMapperSingleton.java @@ -1,6 +1,9 @@ package org.broadinstitute.dsm.util; import com.fasterxml.jackson.databind.ObjectMapper; +import org.broadinstitute.dsm.model.elastic.export.generate.JsonParseException; + +import java.io.IOException; public class ObjectMapperSingleton { @@ -10,6 +13,16 @@ public static ObjectMapper instance() { return Helper.objectMapperInstance; } + public static T readValue(String content, Class valueType) { + try { + return Helper.objectMapperInstance.readValue(content, valueType); + } catch (com.fasterxml.jackson.core.JsonParseException e) { + throw new JsonParseException(e.getMessage()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + private static class Helper { private static final ObjectMapper objectMapperInstance = new ObjectMapper(); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java index 9e6ccf8f0..5b9083849 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java @@ -17,13 +17,13 @@ public static void setUp() { @Test public void testExport() { - final String index = "participants_structured.cmi.angio"; - final String study = "angio"; + final String index = "participants_structured.rgp.rgp"; + final String study = "rgp"; List exportables = Arrays.asList( //DynamicFieldsMappingMigrator should be first in the list to make sure that mapping will be exported for first + new DynamicFieldsMappingMigrator(index, study), new ParticipantDataMigrator(index, study), new ParticipantMigrator(index, study), - new DynamicFieldsMappingMigrator(index, study), new OncHistoryMigrator(index, study), new MedicalRecordMigrator(index, study), new OncHistoryDetailsMigrator(index, study), From 0e8441ff36da70c7e8ca23bd6112afcc978796de Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 6 Jan 2022 19:12:09 +0400 Subject: [PATCH 297/405] DDP-7121 | change IOException to Exception for possibleValuesJson --- .../dsm/model/elastic/export/parse/DynamicFieldsParser.java | 2 +- .../elastic/migration/DynamicFieldsMappingMigratorTest.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/DynamicFieldsParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/DynamicFieldsParser.java index 615864a94..8383705ac 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/DynamicFieldsParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/DynamicFieldsParser.java @@ -98,7 +98,7 @@ private Optional getTypeFromPossibleValuesJson() { .map(possibleValue -> possibleValue.get(TYPE)) .findFirst(); return maybeType; - } catch (IOException e) { + } catch (Exception e) { return Optional.empty(); } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java index 9e6ccf8f0..b39dc355b 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java @@ -17,8 +17,8 @@ public static void setUp() { @Test public void testExport() { - final String index = "participants_structured.cmi.angio"; - final String study = "angio"; + final String index = "participants_structured.atcp.atcp"; + final String study = "atcp"; List exportables = Arrays.asList( //DynamicFieldsMappingMigrator should be first in the list to make sure that mapping will be exported for first new ParticipantDataMigrator(index, study), From 5dc96d3cd2e17bdacc08d3d2dc21382d0dcbb24d Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 6 Jan 2022 19:20:56 +0400 Subject: [PATCH 298/405] DDP-7121 | minor change --- .../elastic/migration/DynamicFieldsMappingMigratorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java index b39dc355b..acbb2522a 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java @@ -21,9 +21,9 @@ public void testExport() { final String study = "atcp"; List exportables = Arrays.asList( //DynamicFieldsMappingMigrator should be first in the list to make sure that mapping will be exported for first + new DynamicFieldsMappingMigrator(index, study), new ParticipantDataMigrator(index, study), new ParticipantMigrator(index, study), - new DynamicFieldsMappingMigrator(index, study), new OncHistoryMigrator(index, study), new MedicalRecordMigrator(index, study), new OncHistoryDetailsMigrator(index, study), From 67086db655884509d0158d91f28c6cc49eae23de Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 10 Jan 2022 15:08:07 +0400 Subject: [PATCH 299/405] DDP-7121 | add changes for fieldsettingsdao --- .../dsm/db/dao/settings/FieldSettingsDao.java | 102 +++++------------- .../export/parse/DynamicFieldsParser.java | 25 ++++- .../dsm/model/elastic/UtilTest.java | 11 +- 3 files changed, 55 insertions(+), 83 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/dao/settings/FieldSettingsDao.java b/src/main/java/org/broadinstitute/dsm/db/dao/settings/FieldSettingsDao.java index 50256b0af..3946d810b 100644 --- a/src/main/java/org/broadinstitute/dsm/db/dao/settings/FieldSettingsDao.java +++ b/src/main/java/org/broadinstitute/dsm/db/dao/settings/FieldSettingsDao.java @@ -69,8 +69,8 @@ public class FieldSettingsDao implements Dao { private static final String SQL_DELETE_FIELD_SETTING_BY_ID = "DELETE FROM field_settings " + "WHERE field_settings_id = ?"; - private static final String SQL_DISPLAY_TYPE_BY_INSTANCE_NAME_AND_COLUMN_NAME = "SELECT display_type FROM field_settings " + - "WHERE ddp_instance_id = (select ddp_instance_id from ddp_instance where instance_name = ?) AND column_name = ?"; + private static final String SQL_DISPLAY_TYPE_BY_INSTANCE_NAME_AND_COLUMN_NAME = GET_FIELD_SETTINGS + + " WHERE ddp_instance_id = (select ddp_instance_id from ddp_instance where instance_name = ?) AND column_name = ?"; private static final String BY_INSTANCE_ID = " WHERE ddp_instance_id = ?"; private static final String AND_BY_COLUMN_NAME = " AND column_name = ?"; @@ -163,7 +163,7 @@ public Optional get(long id) { return Optional.empty(); } - public Optional getDisplayTypeByInstanceNameAndColumnName(String instanceName, String columnName) { + public Optional getFieldSettingsByInstanceNameAndColumnName(String instanceName, String columnName) { SimpleResult simpleResult = inTransaction(conn -> { SimpleResult execResult = new SimpleResult(); try (PreparedStatement stmt = conn.prepareStatement(SQL_DISPLAY_TYPE_BY_INSTANCE_NAME_AND_COLUMN_NAME)) { @@ -171,7 +171,7 @@ public Optional getDisplayTypeByInstanceNameAndColumnName(String instanc stmt.setString(2, columnName); try (ResultSet resultSet = stmt.executeQuery()) { if (resultSet.next()) { - execResult.resultValue = resultSet.getString(DISPLAY_TYPE); + execResult.resultValue = buildFieldSettingsFromResultSet(resultSet); } } } catch (SQLException sqle) { @@ -184,7 +184,7 @@ public Optional getDisplayTypeByInstanceNameAndColumnName(String instanc throw new RuntimeException("could not find the specified display type by instance name and column name", simpleResult.resultException); } - return Optional.ofNullable((String) simpleResult.resultValue); + return Optional.ofNullable((FieldSettingsDto) simpleResult.resultValue); } public List getOptionAndRadioFieldSettingsByInstanceId(int instanceId) { @@ -196,20 +196,7 @@ public List getOptionAndRadioFieldSettingsByInstanceId(int ins try(ResultSet rs = stmt.executeQuery()) { while (rs.next()) { fieldSettingsByOptions.add( - new FieldSettingsDto.Builder(rs.getInt(DDP_INSTANCE_ID)) - .withFieldSettingsId(rs.getInt(FIELD_SETTINGS_ID)) - .withFieldType(rs.getString(FIELD_TYPE)) - .withColumnName(rs.getString(COLUMN_NAME)) - .withColumnDisplay(rs.getString(COLUMN_DISPLAY)) - .withDisplayType(rs.getString(DISPLAY_TYPE)) - .withPossibleValues(rs.getString(POSSIBLE_VALUES)) - .withActions(rs.getString(ACTIONS)) - .withReadOnly(rs.getBoolean(READONLY)) - .withOrderNumber(rs.getInt(ORDER_NUMBER)) - .withDeleted(rs.getBoolean(DELETED)) - .withLastChanged(rs.getLong(LAST_CHANGED)) - .withChangedBy(rs.getString(CHANGED_BY)) - .build() + buildFieldSettingsFromResultSet(rs) ); } } @@ -226,6 +213,23 @@ public List getOptionAndRadioFieldSettingsByInstanceId(int ins return fieldSettingsByOptions; } + private FieldSettingsDto buildFieldSettingsFromResultSet(ResultSet rs) throws SQLException { + return new FieldSettingsDto.Builder(rs.getInt(DDP_INSTANCE_ID)) + .withFieldSettingsId(rs.getInt(FIELD_SETTINGS_ID)) + .withFieldType(rs.getString(FIELD_TYPE)) + .withColumnName(rs.getString(COLUMN_NAME)) + .withColumnDisplay(rs.getString(COLUMN_DISPLAY)) + .withDisplayType(rs.getString(DISPLAY_TYPE)) + .withPossibleValues(rs.getString(POSSIBLE_VALUES)) + .withActions(rs.getString(ACTIONS)) + .withReadOnly(rs.getBoolean(READONLY)) + .withOrderNumber(rs.getInt(ORDER_NUMBER)) + .withDeleted(rs.getBoolean(DELETED)) + .withLastChanged(rs.getLong(LAST_CHANGED)) + .withChangedBy(rs.getString(CHANGED_BY)) + .build(); + } + public List getAllFieldSettings() { List fieldSettingsByOptions = new ArrayList<>(); SimpleResult results = inTransaction((conn) -> { @@ -234,20 +238,7 @@ public List getAllFieldSettings() { try(ResultSet rs = stmt.executeQuery()) { while (rs.next()) { fieldSettingsByOptions.add( - new FieldSettingsDto.Builder(rs.getInt(DDP_INSTANCE_ID)) - .withFieldSettingsId(rs.getInt(FIELD_SETTINGS_ID)) - .withFieldType(rs.getString(FIELD_TYPE)) - .withColumnName(rs.getString(COLUMN_NAME)) - .withColumnDisplay(rs.getString(COLUMN_DISPLAY)) - .withDisplayType(rs.getString(DISPLAY_TYPE)) - .withPossibleValues(rs.getString(POSSIBLE_VALUES)) - .withActions(rs.getString(ACTIONS)) - .withReadOnly(rs.getBoolean(READONLY)) - .withOrderNumber(rs.getInt(ORDER_NUMBER)) - .withDeleted(rs.getBoolean(DELETED)) - .withLastChanged(rs.getLong(LAST_CHANGED)) - .withChangedBy(rs.getString(CHANGED_BY)) - .build() + buildFieldSettingsFromResultSet(rs) ); } } @@ -272,20 +263,7 @@ public List getFieldSettingsByInstanceId(int instanceId) { try(ResultSet fieldSettingsByInstanceIdRs = stmt.executeQuery()) { while (fieldSettingsByInstanceIdRs.next()) { fieldSettingsByOptions.add( - new FieldSettingsDto.Builder(fieldSettingsByInstanceIdRs.getInt(DDP_INSTANCE_ID)) - .withFieldSettingsId(fieldSettingsByInstanceIdRs.getInt(FIELD_SETTINGS_ID)) - .withFieldType(fieldSettingsByInstanceIdRs.getString(FIELD_TYPE)) - .withColumnName(fieldSettingsByInstanceIdRs.getString(COLUMN_NAME)) - .withColumnDisplay(fieldSettingsByInstanceIdRs.getString(COLUMN_DISPLAY)) - .withDisplayType(fieldSettingsByInstanceIdRs.getString(DISPLAY_TYPE)) - .withPossibleValues(fieldSettingsByInstanceIdRs.getString(POSSIBLE_VALUES)) - .withActions(fieldSettingsByInstanceIdRs.getString(ACTIONS)) - .withReadOnly(fieldSettingsByInstanceIdRs.getBoolean(READONLY)) - .withOrderNumber(fieldSettingsByInstanceIdRs.getInt(ORDER_NUMBER)) - .withDeleted(fieldSettingsByInstanceIdRs.getBoolean(DELETED)) - .withLastChanged(fieldSettingsByInstanceIdRs.getLong(LAST_CHANGED)) - .withChangedBy(fieldSettingsByInstanceIdRs.getString(CHANGED_BY)) - .build() + buildFieldSettingsFromResultSet(fieldSettingsByInstanceIdRs) ); } } @@ -309,20 +287,7 @@ public Optional getFieldSettingByColumnNameAndInstanceId(int i stmt.setString(2, columnName); try(ResultSet fieldSettingsByColumnNameRs = stmt.executeQuery()) { if (fieldSettingsByColumnNameRs.next()) { - dbVals.resultValue = new FieldSettingsDto.Builder(fieldSettingsByColumnNameRs.getInt(DDP_INSTANCE_ID)) - .withFieldSettingsId(fieldSettingsByColumnNameRs.getInt(FIELD_SETTINGS_ID)) - .withFieldType(fieldSettingsByColumnNameRs.getString(FIELD_TYPE)) - .withColumnName(fieldSettingsByColumnNameRs.getString(COLUMN_NAME)) - .withColumnDisplay(fieldSettingsByColumnNameRs.getString(COLUMN_DISPLAY)) - .withDisplayType(fieldSettingsByColumnNameRs.getString(DISPLAY_TYPE)) - .withPossibleValues(fieldSettingsByColumnNameRs.getString(POSSIBLE_VALUES)) - .withActions(fieldSettingsByColumnNameRs.getString(ACTIONS)) - .withReadOnly(fieldSettingsByColumnNameRs.getBoolean(READONLY)) - .withOrderNumber(fieldSettingsByColumnNameRs.getInt(ORDER_NUMBER)) - .withDeleted(fieldSettingsByColumnNameRs.getBoolean(DELETED)) - .withLastChanged(fieldSettingsByColumnNameRs.getLong(LAST_CHANGED)) - .withChangedBy(fieldSettingsByColumnNameRs.getString(CHANGED_BY)) - .build(); + dbVals.resultValue = buildFieldSettingsFromResultSet(fieldSettingsByColumnNameRs); } } } @@ -349,20 +314,7 @@ public List getFieldSettingsByInstanceIdAndColumns(int instan try(ResultSet rs = stmt.executeQuery()) { while (rs.next()) { fieldSettingsByColumnNames.add( - new FieldSettingsDto.Builder(rs.getInt(DDP_INSTANCE_ID)) - .withFieldSettingsId(rs.getInt(FIELD_SETTINGS_ID)) - .withFieldType(rs.getString(FIELD_TYPE)) - .withColumnName(rs.getString(COLUMN_NAME)) - .withColumnDisplay(rs.getString(COLUMN_DISPLAY)) - .withDisplayType(rs.getString(DISPLAY_TYPE)) - .withPossibleValues(rs.getString(POSSIBLE_VALUES)) - .withActions(rs.getString(ACTIONS)) - .withReadOnly(rs.getBoolean(READONLY)) - .withOrderNumber(rs.getInt(ORDER_NUMBER)) - .withDeleted(rs.getBoolean(DELETED)) - .withLastChanged(rs.getLong(LAST_CHANGED)) - .withChangedBy(rs.getString(CHANGED_BY)) - .build() + buildFieldSettingsFromResultSet(rs) ); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/DynamicFieldsParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/DynamicFieldsParser.java index 8383705ac..a43dd09ed 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/DynamicFieldsParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/DynamicFieldsParser.java @@ -3,9 +3,9 @@ import com.fasterxml.jackson.core.type.TypeReference; import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.dao.settings.FieldSettingsDao; +import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; import org.broadinstitute.dsm.util.ObjectMapperSingleton; -import java.io.IOException; import java.util.List; import java.util.Map; import java.util.Optional; @@ -45,7 +45,7 @@ public void setFieldSettingsDao(FieldSettingsDao fieldSettingsDao) { public Object parse(String element) { if (StringUtils.isBlank(displayType)) { - displayType = fieldSettingsDao.getDisplayTypeByInstanceNameAndColumnName(realm, super.fieldName).orElse(StringUtils.EMPTY); + getProperDisplayTypeWithPossibleValues(); } Object parsedValue; @@ -57,7 +57,10 @@ public Object parse(String element) { Optional maybeType = getTypeFromPossibleValuesJson(); this.displayType = maybeType.orElse(StringUtils.EMPTY); parsedValue = maybeType - .map(this::parse) + .map(displayType -> { + if (parser instanceof TypeParser) return parse(displayType); + else return parse(element); + }) .orElse(forString(element)); } else if (NUMBER.equals(displayType)) { parsedValue = forNumeric(element); @@ -70,6 +73,22 @@ public Object parse(String element) { return parsedValue; } + private void getProperDisplayTypeWithPossibleValues() { + Optional fieldSettingsByInstanceNameAndColumnName = + fieldSettingsDao.getFieldSettingsByInstanceNameAndColumnName(realm, super.fieldName); + if (fieldSettingsByInstanceNameAndColumnName.isPresent()) { + FieldSettingsDto fieldSettings = fieldSettingsByInstanceNameAndColumnName.get(); + displayType = StringUtils.isNotBlank(fieldSettings.getDisplayType()) + ? fieldSettings.getDisplayType() + : StringUtils.EMPTY; + possibleValuesJson = StringUtils.isNotBlank(fieldSettings.getPossibleValues()) + ? fieldSettings.getPossibleValues() + : StringUtils.EMPTY; + } else { + displayType = StringUtils.EMPTY; + } + } + @Override protected Object forNumeric(String value) { return parser.forNumeric(value); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java index eef0cbde3..f1663a719 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java @@ -9,6 +9,7 @@ import org.broadinstitute.dsm.db.Participant; import org.broadinstitute.dsm.db.dao.settings.FieldSettingsDao; import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; +import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; import org.broadinstitute.dsm.model.FollowUp; import org.junit.Assert; import org.junit.Test; @@ -57,14 +58,14 @@ public void transformJsonToMap() { class FieldSettingsDaoMock extends FieldSettingsDao { @Override - public Optional getDisplayTypeByInstanceNameAndColumnName(String instanceName, String columnName) { - String displayType = StringUtils.EMPTY; + public Optional getFieldSettingsByInstanceNameAndColumnName(String instanceName, String columnName) { + FieldSettingsDto.Builder builder = new FieldSettingsDto.Builder(0); if ("BOOLEAN_VAL".equals(columnName)) { - displayType = "CHECKBOX"; + builder.withDisplayType("CHECKBOX"); } else if ("LONG_VAL".equals(columnName)) { - displayType = "NUMBER"; + builder.withDisplayType("NUMBER"); } - return Optional.of(displayType); + return Optional.of(builder.build()); } } From 0a1fa9a5d9f4832c65aa7fcf12df8d8910df90dc Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 10 Jan 2022 16:01:53 +0400 Subject: [PATCH 300/405] DDP-7121 | reorginize elastic filtering, extracting hardcoded constants --- .../broadinstitute/dsm/model/elastic/Util.java | 3 +++ .../elastic/filter/CollectionQueryBuilder.java | 5 ----- .../elastic/filter/DsmAbstractQueryBuilder.java | 13 ++++++++----- .../dsm/model/elastic/filter/FilterParser.java | 6 +++--- .../dsm/model/elastic/filter/Operator.java | 13 ++++++------- .../elastic/filter/QueryBuilderFactory.java | 2 +- .../dsm/model/elastic/filter/QueryPayload.java | 3 ++- .../elastic/filter/SingleQueryBuilder.java | 7 ------- .../filter/{ => splitter}/BaseSplitter.java | 4 ++-- .../{ => splitter}/DateGreaterSplitter.java | 5 +++-- .../{ => splitter}/DateLowerSplitter.java | 5 +++-- .../filter/{ => splitter}/DateSplitter.java | 8 +++++--- .../{ => splitter}/DiamondEqualsSplitter.java | 6 +++--- .../filter/{ => splitter}/EqualsSplitter.java | 2 +- .../GreaterThanEqualsSplitter.java | 2 +- .../{ => splitter}/IsNotNullSplitter.java | 2 +- .../{ => splitter}/JsonExtractSplitter.java | 17 +++++++++-------- .../{ => splitter}/LessThanEqualsSplitter.java | 2 +- .../filter/{ => splitter}/LikeSplitter.java | 2 +- .../{ => splitter}/MultipleOptionsSplitter.java | 2 +- .../filter/{ => splitter}/SplitterFactory.java | 5 +++-- .../filter/{ => splitter}/StrDateSplitter.java | 5 +++-- .../dsm/util/ElasticSearchUtil.java | 4 ++-- .../filter/AndOrFilterSeparatorTest.java | 7 ------- .../elastic/filter/DateGreaterSplitterTest.java | 2 ++ .../elastic/filter/DateLowerSplitterTest.java | 2 ++ .../model/elastic/filter/DateSplitterTest.java | 4 ++-- .../filter/DiamondEqualsSplitterTest.java | 1 + .../elastic/filter/JsonExtractSplitterTest.java | 3 ++- .../filter/MultipleOptionsSplitterTest.java | 2 ++ .../elastic/filter/StrDateSplitterTest.java | 2 ++ 31 files changed, 75 insertions(+), 71 deletions(-) rename src/main/java/org/broadinstitute/dsm/model/elastic/filter/{ => splitter}/BaseSplitter.java (90%) rename src/main/java/org/broadinstitute/dsm/model/elastic/filter/{ => splitter}/DateGreaterSplitter.java (72%) rename src/main/java/org/broadinstitute/dsm/model/elastic/filter/{ => splitter}/DateLowerSplitter.java (74%) rename src/main/java/org/broadinstitute/dsm/model/elastic/filter/{ => splitter}/DateSplitter.java (51%) rename src/main/java/org/broadinstitute/dsm/model/elastic/filter/{ => splitter}/DiamondEqualsSplitter.java (82%) rename src/main/java/org/broadinstitute/dsm/model/elastic/filter/{ => splitter}/EqualsSplitter.java (76%) rename src/main/java/org/broadinstitute/dsm/model/elastic/filter/{ => splitter}/GreaterThanEqualsSplitter.java (78%) rename src/main/java/org/broadinstitute/dsm/model/elastic/filter/{ => splitter}/IsNotNullSplitter.java (77%) rename src/main/java/org/broadinstitute/dsm/model/elastic/filter/{ => splitter}/JsonExtractSplitter.java (68%) rename src/main/java/org/broadinstitute/dsm/model/elastic/filter/{ => splitter}/LessThanEqualsSplitter.java (77%) rename src/main/java/org/broadinstitute/dsm/model/elastic/filter/{ => splitter}/LikeSplitter.java (76%) rename src/main/java/org/broadinstitute/dsm/model/elastic/filter/{ => splitter}/MultipleOptionsSplitter.java (94%) rename src/main/java/org/broadinstitute/dsm/model/elastic/filter/{ => splitter}/SplitterFactory.java (90%) rename src/main/java/org/broadinstitute/dsm/model/elastic/filter/{ => splitter}/StrDateSplitter.java (81%) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 285508ae5..2d37cf44c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -35,7 +35,10 @@ public class Util { ); public static final int FIRST_ELEMENT_INDEX = 0; public static final String UNDERSCORE_SEPARATOR = "_"; + public static final String COMMA_SEPARATOR = ","; public static final String DOC = "_doc"; + public static final String ESCAPE_CHARACTER = "\\"; + public static final String FORWARD_SLASH_SEPARATOR = "/"; private static final Pattern CAMEL_CASE_REGEX = Pattern.compile("(([a-z])+([A-z])+(\\.)*)*"); private static final Pattern UPPER_CASE_REGEX = Pattern.compile("(?=\\p{Upper})"); public static final Gson GSON = new Gson(); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java index 5c3746614..d0e991231 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java @@ -1,15 +1,10 @@ package org.broadinstitute.dsm.model.elastic.filter; import org.apache.lucene.search.join.ScoreMode; -import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import org.elasticsearch.index.query.NestedQueryBuilder; public class CollectionQueryBuilder extends DsmAbstractQueryBuilder { - public CollectionQueryBuilder(String filter, Parser parser) { - super(filter, parser); - } - public CollectionQueryBuilder() {} @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java index 1c9880a18..1b3555e5e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java @@ -1,17 +1,19 @@ package org.broadinstitute.dsm.model.elastic.filter; +import java.util.List; +import java.util.Map; + import org.broadinstitute.dsm.model.Filter; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; +import org.broadinstitute.dsm.model.elastic.filter.splitter.BaseSplitter; +import org.broadinstitute.dsm.model.elastic.filter.splitter.SplitterFactory; import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.statics.ESObjectConstants; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; -import java.util.List; -import java.util.Map; - public abstract class DsmAbstractQueryBuilder { protected static final String DSM_WITH_DOT = ESObjectConstants.DSM + DBConstants.ALIAS_DELIMITER; @@ -30,9 +32,10 @@ public DsmAbstractQueryBuilder(String filter, Parser parser) { } public static DsmAbstractQueryBuilder of(String alias) { - DsmAbstractQueryBuilder queryBuilder; boolean isCollection = Util.TABLE_ALIAS_MAPPINGS.get(alias).isCollection(); - return isCollection ? new CollectionQueryBuilder() : new SingleQueryBuilder(); + return isCollection + ? new CollectionQueryBuilder() + : new SingleQueryBuilder(); } public DsmAbstractQueryBuilder() { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/FilterParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/FilterParser.java index 1a45895e1..ced1df238 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/FilterParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/FilterParser.java @@ -1,11 +1,11 @@ package org.broadinstitute.dsm.model.elastic.filter; -import org.apache.commons.lang3.StringUtils; -import org.broadinstitute.dsm.model.elastic.export.parse.ValueParser; - import java.util.ArrayList; import java.util.List; +import org.apache.commons.lang3.StringUtils; +import org.broadinstitute.dsm.model.elastic.export.parse.ValueParser; + public class FilterParser extends ValueParser { // '12313' - str diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java index c64423f42..5ee9b3ba3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java @@ -1,11 +1,10 @@ package org.broadinstitute.dsm.model.elastic.filter; +import java.util.Arrays; + import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.Filter; -import java.util.Arrays; -import java.util.stream.Collectors; - public enum Operator { LIKE(Filter.LIKE_TRIMMED), EQUALS(Filter.EQUALS_TRIMMED), @@ -20,6 +19,8 @@ public enum Operator { JSON_EXTRACT(Filter.JSON_EXTRACT), DATE(Filter.DATE); + public static final String UNKNOWN_OPERATOR = "Unknown operator"; + private String value; Operator(String value) { @@ -30,7 +31,7 @@ public static Operator getOperator(String value) { for (Operator op: Operator.values()) { if (op.value.equals(value)) return op; } - throw new IllegalArgumentException("Unknown operator"); + throw new IllegalArgumentException(UNKNOWN_OPERATOR); } public static Operator extract(String filter) { @@ -49,13 +50,11 @@ else if (filter.contains(Filter.DATE_LESS)) return DATE_LESS; else if (filter.contains(Filter.JSON_EXTRACT)) return JSON_EXTRACT; - String operator = Arrays.stream(filter.split(" ")) + String operator = Arrays.stream(filter.split(Filter.SPACE)) .filter(StringUtils::isNotBlank) .filter(str -> Arrays.stream(Operator.values()).anyMatch(op -> op.value.equals(str))) .findFirst() .orElse(StringUtils.EMPTY); - // [NOT , m.mr_problem, <=>, 1] - // return getOperator(operator); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java index 311293f51..618e8dc5b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java @@ -58,7 +58,7 @@ public static QueryBuilder buildQueryBuilder(Operator operator, QueryPayload pay } break; default: - throw new IllegalArgumentException("Unknown operator"); + throw new IllegalArgumentException(Operator.UNKNOWN_OPERATOR); } return qb; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryPayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryPayload.java index f67eaa9f0..5db9ad3a1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryPayload.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryPayload.java @@ -1,6 +1,7 @@ package org.broadinstitute.dsm.model.elastic.filter; import lombok.Getter; +import org.broadinstitute.dsm.statics.DBConstants; @Getter public class QueryPayload { @@ -16,6 +17,6 @@ public QueryPayload(String path, String property, Object[] values) { } public String getFieldName() { - return path + "." + property; + return path + DBConstants.ALIAS_DELIMITER + property; } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SingleQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SingleQueryBuilder.java index 0a6f3b201..9589e975c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SingleQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SingleQueryBuilder.java @@ -1,14 +1,7 @@ package org.broadinstitute.dsm.model.elastic.filter; -import org.broadinstitute.dsm.model.elastic.export.parse.Parser; -import org.elasticsearch.index.query.MatchQueryBuilder; - public class SingleQueryBuilder extends DsmAbstractQueryBuilder { - public SingleQueryBuilder(String filter, Parser parser) { - super(filter, parser); - } - public SingleQueryBuilder() {} @Override diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/BaseSplitter.java similarity index 90% rename from src/main/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitter.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/BaseSplitter.java index a979fe77c..46a1d7e91 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/BaseSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/BaseSplitter.java @@ -1,4 +1,4 @@ -package org.broadinstitute.dsm.model.elastic.filter; +package org.broadinstitute.dsm.model.elastic.filter.splitter; import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.elastic.Util; @@ -27,7 +27,7 @@ public String getInnerProperty() { } protected String[] getFieldWithAlias() { - return splittedFilter[0].trim().split(ElasticSearchUtil.DOT_SEPARATOR); + return splittedFilter[0].trim().split(ElasticSearchUtil.ESCAPE_CHARACTER_DOT_SEPARATOR); } public void setFilter(String filter) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DateGreaterSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateGreaterSplitter.java similarity index 72% rename from src/main/java/org/broadinstitute/dsm/model/elastic/filter/DateGreaterSplitter.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateGreaterSplitter.java index 63669280d..8771e98e4 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DateGreaterSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateGreaterSplitter.java @@ -1,7 +1,8 @@ -package org.broadinstitute.dsm.model.elastic.filter; +package org.broadinstitute.dsm.model.elastic.filter.splitter; import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.Filter; +import org.broadinstitute.dsm.model.elastic.Util; public class DateGreaterSplitter extends GreaterThanEqualsSplitter { @@ -10,7 +11,7 @@ public String[] getValue() { return new String[]{splittedFilter[1].split(Filter.DATE_FORMAT)[1] .replace(Filter.OPEN_PARENTHESIS, StringUtils.EMPTY) .replace(Filter.CLOSE_PARENTHESIS, StringUtils.EMPTY) - .split(",")[0]}; + .split(Util.COMMA_SEPARATOR)[0]}; } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DateLowerSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateLowerSplitter.java similarity index 74% rename from src/main/java/org/broadinstitute/dsm/model/elastic/filter/DateLowerSplitter.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateLowerSplitter.java index ac08402bf..bb08477ea 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DateLowerSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateLowerSplitter.java @@ -1,7 +1,8 @@ -package org.broadinstitute.dsm.model.elastic.filter; +package org.broadinstitute.dsm.model.elastic.filter.splitter; import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.Filter; +import org.broadinstitute.dsm.model.elastic.Util; public class DateLowerSplitter extends LessThanEqualsSplitter { @@ -11,6 +12,6 @@ public String[] getValue() { return new String[]{splittedFilter[1].split(Filter.DATE_FORMAT)[1] .replace(Filter.OPEN_PARENTHESIS, StringUtils.EMPTY) .replace(Filter.CLOSE_PARENTHESIS, StringUtils.EMPTY) - .split(",")[0]}; + .split(Util.COMMA_SEPARATOR)[0]}; } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateSplitter.java similarity index 51% rename from src/main/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitter.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateSplitter.java index 9a32b3187..3fef93046 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateSplitter.java @@ -1,6 +1,7 @@ -package org.broadinstitute.dsm.model.elastic.filter; +package org.broadinstitute.dsm.model.elastic.filter.splitter; import org.broadinstitute.dsm.model.Filter; +import org.broadinstitute.dsm.model.elastic.Util; public class DateSplitter extends EqualsSplitter { @@ -10,8 +11,9 @@ public String[] split() { String[] equalSeparated = super.split(); String leftSide = equalSeparated[0].trim(); String rightSide = equalSeparated[1].trim(); - String fieldWithAlias = leftSide.split("\\" + Filter.OPEN_PARENTHESIS)[2].split("/")[0]; - String value = rightSide.split("\\" + Filter.OPEN_PARENTHESIS)[2].split("\\" + Filter.CLOSE_PARENTHESIS)[0]; + String fieldWithAlias = + leftSide.split(Util.ESCAPE_CHARACTER + Filter.OPEN_PARENTHESIS)[2].split(Util.FORWARD_SLASH_SEPARATOR)[0]; + String value = rightSide.split(Util.ESCAPE_CHARACTER + Filter.OPEN_PARENTHESIS)[2].split(Util.ESCAPE_CHARACTER + Filter.CLOSE_PARENTHESIS)[0]; return new String[] {fieldWithAlias, value}; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DiamondEqualsSplitter.java similarity index 82% rename from src/main/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitter.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DiamondEqualsSplitter.java index f73ce990e..9f30b845a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DiamondEqualsSplitter.java @@ -1,4 +1,4 @@ -package org.broadinstitute.dsm.model.elastic.filter; +package org.broadinstitute.dsm.model.elastic.filter.splitter; import org.broadinstitute.dsm.model.Filter; import org.broadinstitute.dsm.model.elastic.Util; @@ -14,12 +14,12 @@ protected String[] getFieldWithAlias() { } private String[] splitFieldWithAliasBySpace(String[] fieldWithAlias) { - return fieldWithAlias[0].split(" "); + return fieldWithAlias[0].split(Filter.SPACE); } @Override public String[] getValue() { - String value = "'" + super.getValue()[0] + "'"; + String value = Filter.SINGLE_QUOTE + super.getValue()[0] + Filter.SINGLE_QUOTE; try { String not = splitFieldWithAliasBySpace(super.getFieldWithAlias())[0]; return new String[] { not + value }; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/EqualsSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/EqualsSplitter.java similarity index 76% rename from src/main/java/org/broadinstitute/dsm/model/elastic/filter/EqualsSplitter.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/EqualsSplitter.java index 0f5867c6c..ab1add268 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/EqualsSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/EqualsSplitter.java @@ -1,4 +1,4 @@ -package org.broadinstitute.dsm.model.elastic.filter; +package org.broadinstitute.dsm.model.elastic.filter.splitter; import org.broadinstitute.dsm.model.Filter; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/GreaterThanEqualsSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/GreaterThanEqualsSplitter.java similarity index 78% rename from src/main/java/org/broadinstitute/dsm/model/elastic/filter/GreaterThanEqualsSplitter.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/GreaterThanEqualsSplitter.java index c67f557ac..c6a591fd3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/GreaterThanEqualsSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/GreaterThanEqualsSplitter.java @@ -1,4 +1,4 @@ -package org.broadinstitute.dsm.model.elastic.filter; +package org.broadinstitute.dsm.model.elastic.filter.splitter; import org.broadinstitute.dsm.model.Filter; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/IsNotNullSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/IsNotNullSplitter.java similarity index 77% rename from src/main/java/org/broadinstitute/dsm/model/elastic/filter/IsNotNullSplitter.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/IsNotNullSplitter.java index 888fbb8bb..a4fa0cfb2 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/IsNotNullSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/IsNotNullSplitter.java @@ -1,4 +1,4 @@ -package org.broadinstitute.dsm.model.elastic.filter; +package org.broadinstitute.dsm.model.elastic.filter.splitter; import org.broadinstitute.dsm.model.Filter; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonExtractSplitter.java similarity index 68% rename from src/main/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitter.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonExtractSplitter.java index 8ab2ad672..1ccf1eadd 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonExtractSplitter.java @@ -1,8 +1,9 @@ -package org.broadinstitute.dsm.model.elastic.filter; +package org.broadinstitute.dsm.model.elastic.filter.splitter; import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.Filter; import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; @@ -26,8 +27,8 @@ public String[] split() { @Override public String getInnerProperty() { - String[] separatedByDot = getFieldWithAlias()[1].split(ElasticSearchUtil.DOT_SEPARATOR); - return String.join(".", separatedByDot[0], Util.underscoresToCamelCase(separatedByDot[1])); + String[] separatedByDot = getFieldWithAlias()[1].split(ElasticSearchUtil.ESCAPE_CHARACTER_DOT_SEPARATOR); + return String.join(DBConstants.ALIAS_DELIMITER, separatedByDot[0], Util.underscoresToCamelCase(separatedByDot[1])); } @Override @@ -37,15 +38,15 @@ protected String[] getFieldWithAlias() { .replace(Filter.OPEN_PARENTHESIS, StringUtils.EMPTY) .replace(Filter.CLOSE_PARENTHESIS, StringUtils.EMPTY) .trim() - .split(","); - String[] splittedByDot = splittedByJsonExtractAndComma[0].split(ElasticSearchUtil.DOT_SEPARATOR); + .split(Util.COMMA_SEPARATOR); + String[] splittedByDot = splittedByJsonExtractAndComma[0].split(ElasticSearchUtil.ESCAPE_CHARACTER_DOT_SEPARATOR); String alias = splittedByDot[0]; String removedSingleQuotes = splittedByJsonExtractAndComma[1] - .replace("'", StringUtils.EMPTY) + .replace(Filter.SINGLE_QUOTE, StringUtils.EMPTY) .trim(); String innerProperty = removedSingleQuotes - .substring(removedSingleQuotes.indexOf(".")+1); - return new String[] {alias, String.join(".", ESObjectConstants.DYNAMIC_FIELDS, + .substring(removedSingleQuotes.indexOf(DBConstants.ALIAS_DELIMITER)+1); + return new String[] {alias, String.join(DBConstants.ALIAS_DELIMITER, ESObjectConstants.DYNAMIC_FIELDS, innerProperty)}; } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LessThanEqualsSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/LessThanEqualsSplitter.java similarity index 77% rename from src/main/java/org/broadinstitute/dsm/model/elastic/filter/LessThanEqualsSplitter.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/LessThanEqualsSplitter.java index 0a718cf72..de2f48c81 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LessThanEqualsSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/LessThanEqualsSplitter.java @@ -1,4 +1,4 @@ -package org.broadinstitute.dsm.model.elastic.filter; +package org.broadinstitute.dsm.model.elastic.filter.splitter; import org.broadinstitute.dsm.model.Filter; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LikeSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/LikeSplitter.java similarity index 76% rename from src/main/java/org/broadinstitute/dsm/model/elastic/filter/LikeSplitter.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/LikeSplitter.java index 3f34ae3f8..b27e02515 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/LikeSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/LikeSplitter.java @@ -1,4 +1,4 @@ -package org.broadinstitute.dsm.model.elastic.filter; +package org.broadinstitute.dsm.model.elastic.filter.splitter; import org.broadinstitute.dsm.model.Filter; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/MultipleOptionsSplitter.java similarity index 94% rename from src/main/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitter.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/MultipleOptionsSplitter.java index 03799f65c..1cf9975c2 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/MultipleOptionsSplitter.java @@ -1,4 +1,4 @@ -package org.broadinstitute.dsm.model.elastic.filter; +package org.broadinstitute.dsm.model.elastic.filter.splitter; import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.Filter; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/SplitterFactory.java similarity index 90% rename from src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/SplitterFactory.java index 902f37eb8..ae9dc9000 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SplitterFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/SplitterFactory.java @@ -1,7 +1,8 @@ -package org.broadinstitute.dsm.model.elastic.filter; +package org.broadinstitute.dsm.model.elastic.filter.splitter; import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.Filter; +import org.broadinstitute.dsm.model.elastic.filter.Operator; public class SplitterFactory { @@ -51,7 +52,7 @@ public static BaseSplitter createSplitter(Operator operator, String filterValue) splitter = new JsonExtractSplitter(decoratedSplitter); break; default: - throw new IllegalArgumentException("Unknown operator"); + throw new IllegalArgumentException(Operator.UNKNOWN_OPERATOR); } return splitter; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/StrDateSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/StrDateSplitter.java similarity index 81% rename from src/main/java/org/broadinstitute/dsm/model/elastic/filter/StrDateSplitter.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/StrDateSplitter.java index 53bde3dbf..146a23eec 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/StrDateSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/StrDateSplitter.java @@ -1,7 +1,8 @@ -package org.broadinstitute.dsm.model.elastic.filter; +package org.broadinstitute.dsm.model.elastic.filter.splitter; import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.Filter; +import org.broadinstitute.dsm.model.elastic.Util; public class StrDateSplitter extends BaseSplitter { @@ -14,7 +15,7 @@ public String[] split() { splittedFilter[i] = dateFieldWithValue[i].split(Filter.DATE_FORMAT)[1] .replace(Filter.OPEN_PARENTHESIS, StringUtils.EMPTY) .replace(Filter.CLOSE_PARENTHESIS, StringUtils.EMPTY) - .split(",")[0]; + .split(Util.COMMA_SEPARATOR)[0]; } return splittedFilter; } diff --git a/src/main/java/org/broadinstitute/dsm/util/ElasticSearchUtil.java b/src/main/java/org/broadinstitute/dsm/util/ElasticSearchUtil.java index b65a2b780..ed1e699a6 100644 --- a/src/main/java/org/broadinstitute/dsm/util/ElasticSearchUtil.java +++ b/src/main/java/org/broadinstitute/dsm/util/ElasticSearchUtil.java @@ -88,7 +88,7 @@ public class ElasticSearchUtil { public static final String AND = " AND ("; public static final String ES = "ES"; public static final String CLOSING_PARENTHESIS = ")"; - public static final String DOT_SEPARATOR = "\\."; + public static final String ESCAPE_CHARACTER_DOT_SEPARATOR = "\\."; public static final String BY_LEGACY_ALTPIDS = " OR profile.legacyAltPid = "; public static final String BY_LEGACY_SHORTID = " AND profile.legacyShortId = "; public static final String END_OF_DAY = " 23:59:59"; @@ -1516,7 +1516,7 @@ private static String getAnyStudy() { private static String getFieldTypeByFieldName(String name) { String anyStudy = getAnyStudy(); String fields = getFieldsAsString(anyStudy); - String[] fieldsArray = name.split(DOT_SEPARATOR); + String[] fieldsArray = name.split(ESCAPE_CHARACTER_DOT_SEPARATOR); String outerField = fieldsArray[OUTER_FIELD_INDEX]; Gson gson = new Gson(); HashMap fieldsMap = gson.fromJson(fields, HashMap.class); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparatorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparatorTest.java index 2d884f836..aaf7d1f56 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparatorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparatorTest.java @@ -49,23 +49,16 @@ public void parseFiltersByLogicalOperators() { @Test public void parseFiltersByLogicalOperatorsSingle() { - String filter = "AND oD.datePx = '15' "; Map> stringListMap = new AndOrFilterSeparator(filter).parseFiltersByLogicalOperators(); Assert.assertEquals("oD.datePx = '15'", stringListMap.get("AND").get(0)); - } @Test public void parseFiltersByLogicalOperatorsSingle2() { - String filter = " AND NOT m.mr_problem <=> 1 "; Map> stringListMap = new AndOrFilterSeparator(filter).parseFiltersByLogicalOperators(); Assert.assertEquals("NOT m.mr_problem <=> 1", stringListMap.get("AND").get(0)); - - - // - // } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateGreaterSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateGreaterSplitterTest.java index 254d1f83e..8d9f675d7 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateGreaterSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateGreaterSplitterTest.java @@ -1,5 +1,7 @@ package org.broadinstitute.dsm.model.elastic.filter; +import org.broadinstitute.dsm.model.elastic.filter.splitter.BaseSplitter; +import org.broadinstitute.dsm.model.elastic.filter.splitter.SplitterFactory; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateLowerSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateLowerSplitterTest.java index a20fdee7b..92879ef8a 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateLowerSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateLowerSplitterTest.java @@ -1,5 +1,7 @@ package org.broadinstitute.dsm.model.elastic.filter; +import org.broadinstitute.dsm.model.elastic.filter.splitter.BaseSplitter; +import org.broadinstitute.dsm.model.elastic.filter.splitter.SplitterFactory; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitterTest.java index 79604aa0f..b6fb68d69 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitterTest.java @@ -1,10 +1,10 @@ package org.broadinstitute.dsm.model.elastic.filter; +import org.broadinstitute.dsm.model.elastic.filter.splitter.BaseSplitter; +import org.broadinstitute.dsm.model.elastic.filter.splitter.SplitterFactory; import org.junit.Assert; import org.junit.Test; -import static org.junit.Assert.*; - public class DateSplitterTest { @Test diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitterTest.java index 264629df0..8b8191ab0 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitterTest.java @@ -1,5 +1,6 @@ package org.broadinstitute.dsm.model.elastic.filter; +import org.broadinstitute.dsm.model.elastic.filter.splitter.DiamondEqualsSplitter; import org.junit.Test; import static org.junit.Assert.*; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitterTest.java index 98c626b1a..19b0da402 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitterTest.java @@ -1,9 +1,10 @@ package org.broadinstitute.dsm.model.elastic.filter; import org.apache.commons.lang3.StringUtils; +import org.broadinstitute.dsm.model.elastic.filter.splitter.BaseSplitter; +import org.broadinstitute.dsm.model.elastic.filter.splitter.SplitterFactory; import org.junit.Assert; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; public class JsonExtractSplitterTest { diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitterTest.java index dac6247ce..9f334c153 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitterTest.java @@ -2,6 +2,8 @@ import static org.junit.Assert.*; +import org.broadinstitute.dsm.model.elastic.filter.splitter.BaseSplitter; +import org.broadinstitute.dsm.model.elastic.filter.splitter.SplitterFactory; import org.junit.Test; public class MultipleOptionsSplitterTest { diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/StrDateSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/StrDateSplitterTest.java index 8b6262180..5681995ba 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/StrDateSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/StrDateSplitterTest.java @@ -1,5 +1,7 @@ package org.broadinstitute.dsm.model.elastic.filter; +import org.broadinstitute.dsm.model.elastic.filter.splitter.BaseSplitter; +import org.broadinstitute.dsm.model.elastic.filter.splitter.SplitterFactory; import org.junit.Assert; import org.junit.Test; From 82dc8ceeddbd122d4b463ec90f49e14e829bbdda Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 10 Jan 2022 16:11:54 +0400 Subject: [PATCH 301/405] DDP-7121 | move tests to appropriate packages, move query building related classes to the package called "query" --- .../dsm/model/elastic/filter/AndOrFilterSeparator.java | 2 +- .../elastic/filter/{ => query}/CollectionQueryBuilder.java | 3 ++- .../elastic/filter/{ => query}/DsmAbstractQueryBuilder.java | 5 ++++- .../elastic/filter/{ => query}/QueryBuilderFactory.java | 3 ++- .../dsm/model/elastic/filter/{ => query}/QueryPayload.java | 2 +- .../elastic/filter/{ => query}/SingleQueryBuilder.java | 4 +++- .../dsm/model/participant/ParticipantWrapper.java | 4 +--- .../filter/{ => query}/CollectionQueryBuilderTest.java | 4 +++- .../elastic/filter/{ => query}/QueryBuilderFactoryTest.java | 5 ++++- .../elastic/filter/{ => query}/SingleQueryBuilderTest.java | 6 +++--- .../filter/{ => splitter}/AndOrFilterSeparatorTest.java | 3 ++- .../filter/{ => splitter}/DateGreaterSplitterTest.java | 3 ++- .../filter/{ => splitter}/DateLowerSplitterTest.java | 3 ++- .../elastic/filter/{ => splitter}/DateSplitterTest.java | 3 ++- .../filter/{ => splitter}/DiamondEqualsSplitterTest.java | 2 +- .../elastic/filter/{ => splitter}/FilterParserTest.java | 3 ++- .../filter/{ => splitter}/JsonExtractSplitterTest.java | 3 ++- .../filter/{ => splitter}/MultipleOptionsSplitterTest.java | 3 ++- .../elastic/filter/{ => splitter}/StrDateSplitterTest.java | 3 ++- 19 files changed, 41 insertions(+), 23 deletions(-) rename src/main/java/org/broadinstitute/dsm/model/elastic/filter/{ => query}/CollectionQueryBuilder.java (77%) rename src/main/java/org/broadinstitute/dsm/model/elastic/filter/{ => query}/DsmAbstractQueryBuilder.java (92%) rename src/main/java/org/broadinstitute/dsm/model/elastic/filter/{ => query}/QueryBuilderFactory.java (95%) rename src/main/java/org/broadinstitute/dsm/model/elastic/filter/{ => query}/QueryPayload.java (88%) rename src/main/java/org/broadinstitute/dsm/model/elastic/filter/{ => query}/SingleQueryBuilder.java (66%) rename src/test/java/org/broadinstitute/dsm/model/elastic/filter/{ => query}/CollectionQueryBuilderTest.java (97%) rename src/test/java/org/broadinstitute/dsm/model/elastic/filter/{ => query}/QueryBuilderFactoryTest.java (78%) rename src/test/java/org/broadinstitute/dsm/model/elastic/filter/{ => query}/SingleQueryBuilderTest.java (81%) rename src/test/java/org/broadinstitute/dsm/model/elastic/filter/{ => splitter}/AndOrFilterSeparatorTest.java (96%) rename src/test/java/org/broadinstitute/dsm/model/elastic/filter/{ => splitter}/DateGreaterSplitterTest.java (82%) rename src/test/java/org/broadinstitute/dsm/model/elastic/filter/{ => splitter}/DateLowerSplitterTest.java (82%) rename src/test/java/org/broadinstitute/dsm/model/elastic/filter/{ => splitter}/DateSplitterTest.java (83%) rename src/test/java/org/broadinstitute/dsm/model/elastic/filter/{ => splitter}/DiamondEqualsSplitterTest.java (94%) rename src/test/java/org/broadinstitute/dsm/model/elastic/filter/{ => splitter}/FilterParserTest.java (86%) rename src/test/java/org/broadinstitute/dsm/model/elastic/filter/{ => splitter}/JsonExtractSplitterTest.java (91%) rename src/test/java/org/broadinstitute/dsm/model/elastic/filter/{ => splitter}/MultipleOptionsSplitterTest.java (86%) rename src/test/java/org/broadinstitute/dsm/model/elastic/filter/{ => splitter}/StrDateSplitterTest.java (85%) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java index be2a8154c..735d3ddab 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java @@ -27,7 +27,7 @@ public void setFilter(String filter) { this.filter = filter; } - protected Map> parseFiltersByLogicalOperators() { + public Map> parseFiltersByLogicalOperators() { Map> filterByLogicalOperators = new ConcurrentHashMap<>(Map.of(Filter.AND_TRIMMED, new ArrayList<>(), Filter.OR_TRIMMED, new ArrayList<>())); int andIndex = filter.indexOf(Filter.AND_TRIMMED); int orIndex = filter.indexOf(Filter.OR_TRIMMED); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/CollectionQueryBuilder.java similarity index 77% rename from src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/CollectionQueryBuilder.java index d0e991231..6b9091422 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/CollectionQueryBuilder.java @@ -1,6 +1,7 @@ -package org.broadinstitute.dsm.model.elastic.filter; +package org.broadinstitute.dsm.model.elastic.filter.query; import org.apache.lucene.search.join.ScoreMode; +import org.broadinstitute.dsm.model.elastic.filter.FilterStrategy; import org.elasticsearch.index.query.NestedQueryBuilder; public class CollectionQueryBuilder extends DsmAbstractQueryBuilder { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/DsmAbstractQueryBuilder.java similarity index 92% rename from src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/DsmAbstractQueryBuilder.java index 1b3555e5e..684d96d0e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/DsmAbstractQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/DsmAbstractQueryBuilder.java @@ -1,4 +1,4 @@ -package org.broadinstitute.dsm.model.elastic.filter; +package org.broadinstitute.dsm.model.elastic.filter.query; import java.util.List; import java.util.Map; @@ -6,6 +6,9 @@ import org.broadinstitute.dsm.model.Filter; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; +import org.broadinstitute.dsm.model.elastic.filter.AndOrFilterSeparator; +import org.broadinstitute.dsm.model.elastic.filter.FilterStrategy; +import org.broadinstitute.dsm.model.elastic.filter.Operator; import org.broadinstitute.dsm.model.elastic.filter.splitter.BaseSplitter; import org.broadinstitute.dsm.model.elastic.filter.splitter.SplitterFactory; import org.broadinstitute.dsm.statics.DBConstants; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryBuilderFactory.java similarity index 95% rename from src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryBuilderFactory.java index 618e8dc5b..32af7434c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryBuilderFactory.java @@ -1,6 +1,7 @@ -package org.broadinstitute.dsm.model.elastic.filter; +package org.broadinstitute.dsm.model.elastic.filter.query; import org.apache.commons.lang3.StringUtils; +import org.broadinstitute.dsm.model.elastic.filter.Operator; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.ExistsQueryBuilder; import org.elasticsearch.index.query.MatchQueryBuilder; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryPayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryPayload.java similarity index 88% rename from src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryPayload.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryPayload.java index 5db9ad3a1..88b6cf36e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/QueryPayload.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryPayload.java @@ -1,4 +1,4 @@ -package org.broadinstitute.dsm.model.elastic.filter; +package org.broadinstitute.dsm.model.elastic.filter.query; import lombok.Getter; import org.broadinstitute.dsm.statics.DBConstants; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SingleQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/SingleQueryBuilder.java similarity index 66% rename from src/main/java/org/broadinstitute/dsm/model/elastic/filter/SingleQueryBuilder.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/SingleQueryBuilder.java index 9589e975c..92a1a8f2e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/SingleQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/SingleQueryBuilder.java @@ -1,4 +1,6 @@ -package org.broadinstitute.dsm.model.elastic.filter; +package org.broadinstitute.dsm.model.elastic.filter.query; + +import org.broadinstitute.dsm.model.elastic.filter.FilterStrategy; public class SingleQueryBuilder extends DsmAbstractQueryBuilder { diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java index 63eca1f27..4939eb37a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java @@ -9,9 +9,8 @@ import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; -import org.broadinstitute.dsm.model.elastic.ESDsm; import org.broadinstitute.dsm.model.elastic.ESProfile; -import org.broadinstitute.dsm.model.elastic.filter.DsmAbstractQueryBuilder; +import org.broadinstitute.dsm.model.elastic.filter.query.DsmAbstractQueryBuilder; import org.broadinstitute.dsm.model.elastic.filter.FilterParser; import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; @@ -20,7 +19,6 @@ import org.broadinstitute.dsm.model.at.DefaultValues; import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; -import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.BoolQueryBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/CollectionQueryBuilderTest.java similarity index 97% rename from src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java rename to src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/CollectionQueryBuilderTest.java index 9fe075723..ee4e5d792 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/CollectionQueryBuilderTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/CollectionQueryBuilderTest.java @@ -1,8 +1,10 @@ -package org.broadinstitute.dsm.model.elastic.filter; +package org.broadinstitute.dsm.model.elastic.filter.query; import java.util.regex.Pattern; import org.apache.lucene.search.join.ScoreMode; +import org.broadinstitute.dsm.model.elastic.filter.FilterParser; +import org.broadinstitute.dsm.model.elastic.filter.query.CollectionQueryBuilder; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.ExistsQueryBuilder; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactoryTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryBuilderFactoryTest.java similarity index 78% rename from src/test/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactoryTest.java rename to src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryBuilderFactoryTest.java index 40ad0d553..6029c1eb7 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/QueryBuilderFactoryTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryBuilderFactoryTest.java @@ -1,7 +1,10 @@ -package org.broadinstitute.dsm.model.elastic.filter; +package org.broadinstitute.dsm.model.elastic.filter.query; import static org.junit.Assert.*; +import org.broadinstitute.dsm.model.elastic.filter.Operator; +import org.broadinstitute.dsm.model.elastic.filter.query.QueryBuilderFactory; +import org.broadinstitute.dsm.model.elastic.filter.query.QueryPayload; import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.RangeQueryBuilder; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/SingleQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/SingleQueryBuilderTest.java similarity index 81% rename from src/test/java/org/broadinstitute/dsm/model/elastic/filter/SingleQueryBuilderTest.java rename to src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/SingleQueryBuilderTest.java index 3227142ab..9008240d7 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/SingleQueryBuilderTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/SingleQueryBuilderTest.java @@ -1,5 +1,7 @@ -package org.broadinstitute.dsm.model.elastic.filter; +package org.broadinstitute.dsm.model.elastic.filter.query; +import org.broadinstitute.dsm.model.elastic.filter.FilterParser; +import org.broadinstitute.dsm.model.elastic.filter.query.SingleQueryBuilder; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.MatchQueryBuilder; @@ -7,8 +9,6 @@ import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.*; - public class SingleQueryBuilderTest { SingleQueryBuilder singleQueryBuilder; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparatorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/AndOrFilterSeparatorTest.java similarity index 96% rename from src/test/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparatorTest.java rename to src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/AndOrFilterSeparatorTest.java index aaf7d1f56..3966697fe 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparatorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/AndOrFilterSeparatorTest.java @@ -1,9 +1,10 @@ -package org.broadinstitute.dsm.model.elastic.filter; +package org.broadinstitute.dsm.model.elastic.filter.splitter; import java.util.ArrayList; import java.util.List; import java.util.Map; +import org.broadinstitute.dsm.model.elastic.filter.AndOrFilterSeparator; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateGreaterSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateGreaterSplitterTest.java similarity index 82% rename from src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateGreaterSplitterTest.java rename to src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateGreaterSplitterTest.java index 8d9f675d7..6d395c56b 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateGreaterSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateGreaterSplitterTest.java @@ -1,5 +1,6 @@ -package org.broadinstitute.dsm.model.elastic.filter; +package org.broadinstitute.dsm.model.elastic.filter.splitter; +import org.broadinstitute.dsm.model.elastic.filter.Operator; import org.broadinstitute.dsm.model.elastic.filter.splitter.BaseSplitter; import org.broadinstitute.dsm.model.elastic.filter.splitter.SplitterFactory; import org.junit.Assert; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateLowerSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateLowerSplitterTest.java similarity index 82% rename from src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateLowerSplitterTest.java rename to src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateLowerSplitterTest.java index 92879ef8a..52fda8163 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateLowerSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateLowerSplitterTest.java @@ -1,5 +1,6 @@ -package org.broadinstitute.dsm.model.elastic.filter; +package org.broadinstitute.dsm.model.elastic.filter.splitter; +import org.broadinstitute.dsm.model.elastic.filter.Operator; import org.broadinstitute.dsm.model.elastic.filter.splitter.BaseSplitter; import org.broadinstitute.dsm.model.elastic.filter.splitter.SplitterFactory; import org.junit.Assert; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateSplitterTest.java similarity index 83% rename from src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitterTest.java rename to src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateSplitterTest.java index b6fb68d69..fb3768d4d 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DateSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateSplitterTest.java @@ -1,5 +1,6 @@ -package org.broadinstitute.dsm.model.elastic.filter; +package org.broadinstitute.dsm.model.elastic.filter.splitter; +import org.broadinstitute.dsm.model.elastic.filter.Operator; import org.broadinstitute.dsm.model.elastic.filter.splitter.BaseSplitter; import org.broadinstitute.dsm.model.elastic.filter.splitter.SplitterFactory; import org.junit.Assert; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DiamondEqualsSplitterTest.java similarity index 94% rename from src/test/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitterTest.java rename to src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DiamondEqualsSplitterTest.java index 8b8191ab0..4a3de4b0f 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/DiamondEqualsSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DiamondEqualsSplitterTest.java @@ -1,4 +1,4 @@ -package org.broadinstitute.dsm.model.elastic.filter; +package org.broadinstitute.dsm.model.elastic.filter.splitter; import org.broadinstitute.dsm.model.elastic.filter.splitter.DiamondEqualsSplitter; import org.junit.Test; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/FilterParserTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/FilterParserTest.java similarity index 86% rename from src/test/java/org/broadinstitute/dsm/model/elastic/filter/FilterParserTest.java rename to src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/FilterParserTest.java index a92ef3ab3..096b7c246 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/FilterParserTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/FilterParserTest.java @@ -1,6 +1,7 @@ -package org.broadinstitute.dsm.model.elastic.filter; +package org.broadinstitute.dsm.model.elastic.filter.splitter; import org.broadinstitute.dsm.model.elastic.export.parse.BaseParser; +import org.broadinstitute.dsm.model.elastic.filter.FilterParser; import org.junit.Assert; import org.junit.Test; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonExtractSplitterTest.java similarity index 91% rename from src/test/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitterTest.java rename to src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonExtractSplitterTest.java index 19b0da402..4b7170ae0 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/JsonExtractSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonExtractSplitterTest.java @@ -1,6 +1,7 @@ -package org.broadinstitute.dsm.model.elastic.filter; +package org.broadinstitute.dsm.model.elastic.filter.splitter; import org.apache.commons.lang3.StringUtils; +import org.broadinstitute.dsm.model.elastic.filter.Operator; import org.broadinstitute.dsm.model.elastic.filter.splitter.BaseSplitter; import org.broadinstitute.dsm.model.elastic.filter.splitter.SplitterFactory; import org.junit.Assert; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/MultipleOptionsSplitterTest.java similarity index 86% rename from src/test/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitterTest.java rename to src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/MultipleOptionsSplitterTest.java index 9f334c153..b75fcd13f 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/MultipleOptionsSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/MultipleOptionsSplitterTest.java @@ -1,7 +1,8 @@ -package org.broadinstitute.dsm.model.elastic.filter; +package org.broadinstitute.dsm.model.elastic.filter.splitter; import static org.junit.Assert.*; +import org.broadinstitute.dsm.model.elastic.filter.Operator; import org.broadinstitute.dsm.model.elastic.filter.splitter.BaseSplitter; import org.broadinstitute.dsm.model.elastic.filter.splitter.SplitterFactory; import org.junit.Test; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/StrDateSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/StrDateSplitterTest.java similarity index 85% rename from src/test/java/org/broadinstitute/dsm/model/elastic/filter/StrDateSplitterTest.java rename to src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/StrDateSplitterTest.java index 5681995ba..2965d9d5d 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/StrDateSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/StrDateSplitterTest.java @@ -1,5 +1,6 @@ -package org.broadinstitute.dsm.model.elastic.filter; +package org.broadinstitute.dsm.model.elastic.filter.splitter; +import org.broadinstitute.dsm.model.elastic.filter.Operator; import org.broadinstitute.dsm.model.elastic.filter.splitter.BaseSplitter; import org.broadinstitute.dsm.model.elastic.filter.splitter.SplitterFactory; import org.junit.Assert; From d895f0cdd56ccbe54dcf0937275714cd5fd7e45b Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 10 Jan 2022 16:55:26 +0400 Subject: [PATCH 302/405] DDP-7121 | remove commented out code --- .../model/participant/ParticipantWrapper.java | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java index 4939eb37a..2fe1b2c6d 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java @@ -137,25 +137,8 @@ private void fetchAndPrepareData(DDPInstance ddpInstance) { if (DDPInstanceDao.getRole(ddpInstance.getName(), DBConstants.KIT_REQUEST_ACTIVATED)) { kitRequests = getKitRequestsFromEsData(); } -// abstractionActivities = -// AbstractionActivity.getAllAbstractionActivityByParticipantIds(ddpInstance.getName(), participantIds); -// abstractionSummary = AbstractionFinal.getAbstractionFinalByParticipantIds(ddpInstance.getName(), participantIds); proxiesByParticipantIds = getProxiesWithParticipantIdsFromElasticList(ddpInstance.getUsersIndexES(), esData.getEsParticipants()); participantData = getParticipantDataFromEsData(); -// List participantIds = getParticipantIdsFromElasticList(esData.getEsParticipants()); -// participants = Participant.getParticipantsByIds(ddpInstance.getName(), participantIds); -// if (ddpInstance.isHasRole()) { -// medicalRecords = MedicalRecord.getMedicalRecordsByParticipantIds(ddpInstance.getName(), participantIds); -// oncHistoryDetails = OncHistoryDetail.getOncHistoryDetailsByParticipantIds(ddpInstance.getName(), participantIds); -// } -// if (DDPInstanceDao.getRole(ddpInstance.getName(), DBConstants.KIT_REQUEST_ACTIVATED)) { -// kitRequests = KitRequestShipping.getKitRequestsByParticipantIds(ddpInstance, participantIds); -// } -// abstractionActivities = -// AbstractionActivity.getAllAbstractionActivityByParticipantIds(ddpInstance.getName(), participantIds); -// abstractionSummary = AbstractionFinal.getAbstractionFinalByParticipantIds(ddpInstance.getName(), participantIds); -// proxiesByParticipantIds = getProxiesWithParticipantIdsFromElasticList(ddpInstance.getUsersIndexES(), esData.getEsParticipants()); -// participantData = new ParticipantDataDao().getParticipantDataByParticipantIds(participantIds); } private Map> getParticipantDataFromEsData() { From e818a8f2fc243d37153025768cde281853a06ee8 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 11 Jan 2022 17:18:07 +0400 Subject: [PATCH 303/405] DDP-7443 | start TDD for multiple field name/values --- .../export/generate/GeneratorPayload.java | 2 ++ .../export/generate/SourceGeneratorTest.java | 33 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorPayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorPayload.java index 08be74550..fa0048033 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorPayload.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/GeneratorPayload.java @@ -10,6 +10,8 @@ public class GeneratorPayload { private String parent; private String parentId; + public GeneratorPayload() {} + public GeneratorPayload(NameValue nameValue, int recordId) { this.nameValue = nameValue; this.recordId = recordId; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorTest.java index 3bdd573bb..bc4d5f5bb 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorTest.java @@ -1,5 +1,6 @@ package org.broadinstitute.dsm.model.elastic.export.generate; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Optional; @@ -60,6 +61,24 @@ public void generateFromJson() { maybeDdpInstance.ifPresentOrElse(m -> Assert.assertEquals("TEST", m.get(Util.underscoresToCamelCase("DDP_INSTANCE"))), Assert::fail); } + @Test + public void generateFromMultipleNameValues() { + final String externalOrderNumber = "externalOrderNumber"; + final String bspCollaboratorSampleId = "bspCollaboratorSampleId"; + List nameValues = Arrays.asList(new NameValue(externalOrderNumber, 12), new NameValue(bspCollaboratorSampleId, 55)); + TestGeneratorPayload testGeneratorPayload = new TestGeneratorPayload(nameValues, 1); + SourceGenerator sourceGenerator = new SourceGenerator(new ValueParser(), testGeneratorPayload); + Map generatedMap = sourceGenerator.generate(); + // dsm.kitRequestShipping.[{}] + List> kitRequestShippings = (List>) ((Map) generatedMap.get("dsm")).get("kitRequestShipping"); + for (Map kitRequestShipping : kitRequestShippings) { + Object externalOrderNumberVal = kitRequestShipping.get(externalOrderNumber); + Object collaboratorSampleIdVal = kitRequestShipping.get(bspCollaboratorSampleId); + Assert.assertEquals("12", externalOrderNumberVal); + Assert.assertEquals("55", collaboratorSampleIdVal); + } + } + private static class TestSourceGenerator extends CollectionSourceGenerator { public TestSourceGenerator(Parser parser, GeneratorPayload generatorPayload) { @@ -72,5 +91,19 @@ protected DBElement getDBElement() { } } + private class TestGeneratorPayload extends GeneratorPayload { + + List nameValue; + + public TestGeneratorPayload(List nameValues, int id) { + this.nameValue = nameValues; + this.recordId = id; + } + + + public List getNameValue() { + return nameValue; + } + } } From 07f59e20136c9f138246151aa59715f69b08d308 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 12 Jan 2022 10:13:58 +0400 Subject: [PATCH 304/405] DDP-7121 | add onc history migrator for pubsub, add medicalRecords to ESDsm --- src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java | 3 +++ .../org/broadinstitute/dsm/pubsub/DSMtasksSubscription.java | 1 + 2 files changed, 4 insertions(+) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java index f010387f3..b05ffb40a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java @@ -2,6 +2,7 @@ import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; import com.google.gson.annotations.SerializedName; @@ -58,6 +59,8 @@ public class ESDsm { OncHistory oncHistory; + List> medicalRecords; + public List getTissue() { if (tissue == null) tissue = Collections.emptyList(); return tissue; diff --git a/src/main/java/org/broadinstitute/dsm/pubsub/DSMtasksSubscription.java b/src/main/java/org/broadinstitute/dsm/pubsub/DSMtasksSubscription.java index bdd571895..68f7a737c 100644 --- a/src/main/java/org/broadinstitute/dsm/pubsub/DSMtasksSubscription.java +++ b/src/main/java/org/broadinstitute/dsm/pubsub/DSMtasksSubscription.java @@ -108,6 +108,7 @@ private static void migrateToES(ExportToES.ExportPayload exportPayload) { new DynamicFieldsMappingMigrator(index, study), new MedicalRecordMigrator(index, study), new OncHistoryDetailsMigrator(index, study), + new OncHistoryMigrator(index, study), new ParticipantDataMigrator(index, study), new ParticipantMigrator(index, study), new KitRequestShippingMigrator(index, study), From 7668fe9c31a7e60ab030b51c9b8c083f72153186 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 12 Jan 2022 19:17:25 +0400 Subject: [PATCH 305/405] DDP-7121 | minor changes in names --- src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java | 2 +- .../dsm/model/elastic/search/ElasticSearchParticipantDto.java | 3 +++ .../elastic/migration/DynamicFieldsMappingMigratorTest.java | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java index b05ffb40a..0ad4d8169 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java @@ -43,7 +43,7 @@ public class ESDsm { @SerializedName(ESObjectConstants.TISSUE_RECORDS) List tissue; - @SerializedName(ESObjectConstants.MEDICAL_RECORDS) + @SerializedName(ESObjectConstants.MEDICAL_RECORD) List medicalRecord; @SerializedName(ESObjectConstants.ONC_HISTORY_DETAIL_RECORDS) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchParticipantDto.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchParticipantDto.java index 83c422c86..35ede491b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchParticipantDto.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchParticipantDto.java @@ -1,6 +1,8 @@ package org.broadinstitute.dsm.model.elastic.search; import java.util.Collections; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.Setter; import org.apache.commons.lang3.StringUtils; @@ -17,6 +19,7 @@ import org.broadinstitute.dsm.model.elastic.ESProfile; @Setter +@JsonIgnoreProperties(ignoreUnknown = true) public class ElasticSearchParticipantDto { private ESAddress address; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java index acbb2522a..7c173deff 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java @@ -17,8 +17,8 @@ public static void setUp() { @Test public void testExport() { - final String index = "participants_structured.atcp.atcp"; - final String study = "atcp"; + final String index = "participants_structured.cmi.angio"; + final String study = "angio"; List exportables = Arrays.asList( //DynamicFieldsMappingMigrator should be first in the list to make sure that mapping will be exported for first new DynamicFieldsMappingMigrator(index, study), From b41abe63ce270bd793cdda6eb827131774b9aea6 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 13 Jan 2022 11:05:48 +0400 Subject: [PATCH 306/405] DDP-7121 | change code to avoid exporting to es for every possible case --- .../dsm/model/patch/BasePatch.java | 16 +++++++++++----- .../dsm/model/patch/PatchFactory.java | 11 +++++++++++ 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index 1679fe3b7..b33c0335f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -50,6 +50,7 @@ public abstract class BasePatch { protected ESProfile profile; protected DDPInstance ddpInstance; protected DBElement dbElement; + private ExportFacade exportFacade; { @@ -65,11 +66,13 @@ protected BasePatch(Patch patch) { prepareCommonData(); } + public void setExportFacade(ExportFacade exportFacade) { + this.exportFacade = exportFacade; + } + private void exportToES(NameValue nameValue) { - GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, Integer.parseInt(patch.getId()), patch.getParent(), patch.getParentId()); - ExportFacadePayload exportFacadePayload = - new ExportFacadePayload(ddpInstance.getParticipantIndexES(), patch.getDdpParticipantId(), generatorPayload, patch.getRealm()); - ExportFacade exportFacade = new ExportFacade(exportFacadePayload); + if (Objects.isNull(exportFacade)) return; + exportFacade.export(); } @@ -83,8 +86,10 @@ private void exportToES(NameValue nameValue) { protected void prepareCommonData() { ddpInstance = DDPInstance.getDDPInstance(patch.getRealm()); - profile = ElasticSearchUtil.getParticipantProfileByGuidOrAltPid(ddpInstance.getParticipantIndexES(), patch.getDdpParticipantId()) + if (Objects.nonNull(exportFacade)){ + profile = ElasticSearchUtil.getParticipantProfileByGuidOrAltPid(ddpInstance.getParticipantIndexES(), patch.getDdpParticipantId()) .orElse(null); + } } Optional processSingleNameValue() { @@ -114,6 +119,7 @@ List processMultipleNameValues() { } protected void exportToESWithId(String id, NameValue nameValue) { + if (Objects.isNull(exportFacade)) return; patch.setId(Objects.requireNonNull(id)); exportToES(Objects.requireNonNull(nameValue)); } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java index bc34f14a8..66a2cd34b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java @@ -1,6 +1,10 @@ package org.broadinstitute.dsm.model.patch; import org.apache.commons.lang3.StringUtils; +import org.broadinstitute.dsm.model.NameValue; +import org.broadinstitute.dsm.model.elastic.export.ExportFacade; +import org.broadinstitute.dsm.model.elastic.export.ExportFacadePayload; +import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; import org.broadinstitute.dsm.util.NotificationUtil; public class PatchFactory { @@ -27,6 +31,13 @@ public static BasePatch makePatch(Patch patch, NotificationUtil notificationUtil if (patcher instanceof NullPatch) { throw new RuntimeException("Id and parentId was null"); } + if (isElasticSearchExportable(patch)) { + NameValue nameValue = patch.getNameValue(); + GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, Integer.parseInt(patch.getId()), patch.getParent(), patch.getParentId()); + ExportFacadePayload exportFacadePayload = + new ExportFacadePayload(ddpInstance.getParticipantIndexES(), patch.getDdpParticipantId(), generatorPayload, patch.getRealm()); + ExportFacade exportFacade = new ExportFacade(exportFacadePayload); + } return patcher; } From cc9ad75e473b9f57cd6a91b1e4999f43f42de510 Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 13 Jan 2022 15:07:54 +0400 Subject: [PATCH 307/405] DDP-7121 | export to ES only in appropriate cases --- .../dsm/model/patch/BasePatch.java | 18 ++++++++++-------- .../dsm/model/patch/PatchFactory.java | 16 +++++++++------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java index b33c0335f..53dda4787 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/BasePatch.java @@ -50,8 +50,7 @@ public abstract class BasePatch { protected ESProfile profile; protected DDPInstance ddpInstance; protected DBElement dbElement; - private ExportFacade exportFacade; - + private boolean isElasticSearchExportable; { resultMap = new HashMap<>(); @@ -66,13 +65,16 @@ protected BasePatch(Patch patch) { prepareCommonData(); } - public void setExportFacade(ExportFacade exportFacade) { - this.exportFacade = exportFacade; + public void setElasticSearchExportable(boolean elasticSearchExportable) { + isElasticSearchExportable = elasticSearchExportable; } private void exportToES(NameValue nameValue) { - if (Objects.isNull(exportFacade)) return; - + if (!isElasticSearchExportable) return; + GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, Integer.parseInt(patch.getId()), patch.getParent(), patch.getParentId()); + ExportFacadePayload exportFacadePayload = + new ExportFacadePayload(ddpInstance.getParticipantIndexES(), patch.getDdpParticipantId(), generatorPayload, patch.getRealm()); + ExportFacade exportFacade = new ExportFacade(exportFacadePayload); exportFacade.export(); } @@ -86,7 +88,7 @@ private void exportToES(NameValue nameValue) { protected void prepareCommonData() { ddpInstance = DDPInstance.getDDPInstance(patch.getRealm()); - if (Objects.nonNull(exportFacade)){ + if (isElasticSearchExportable){ profile = ElasticSearchUtil.getParticipantProfileByGuidOrAltPid(ddpInstance.getParticipantIndexES(), patch.getDdpParticipantId()) .orElse(null); } @@ -119,7 +121,7 @@ List processMultipleNameValues() { } protected void exportToESWithId(String id, NameValue nameValue) { - if (Objects.isNull(exportFacade)) return; + if (!isElasticSearchExportable) return; patch.setId(Objects.requireNonNull(id)); exportToES(Objects.requireNonNull(nameValue)); } diff --git a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java index 66a2cd34b..1502c2cef 100644 --- a/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/patch/PatchFactory.java @@ -2,11 +2,14 @@ import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.NameValue; +import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.ExportFacade; import org.broadinstitute.dsm.model.elastic.export.ExportFacadePayload; import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; import org.broadinstitute.dsm.util.NotificationUtil; +import java.util.Objects; + public class PatchFactory { public static BasePatch makePatch(Patch patch, NotificationUtil notificationUtil) { @@ -31,16 +34,15 @@ public static BasePatch makePatch(Patch patch, NotificationUtil notificationUtil if (patcher instanceof NullPatch) { throw new RuntimeException("Id and parentId was null"); } - if (isElasticSearchExportable(patch)) { - NameValue nameValue = patch.getNameValue(); - GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, Integer.parseInt(patch.getId()), patch.getParent(), patch.getParentId()); - ExportFacadePayload exportFacadePayload = - new ExportFacadePayload(ddpInstance.getParticipantIndexES(), patch.getDdpParticipantId(), generatorPayload, patch.getRealm()); - ExportFacade exportFacade = new ExportFacade(exportFacadePayload); - } + patcher.setElasticSearchExportable(isElasticSearchExportable(patch)); return patcher; } + private static boolean isElasticSearchExportable(Patch patch) { + if (Objects.isNull(patch.getTableAlias())) return false; + return Util.TABLE_ALIAS_MAPPINGS.containsKey(patch.getTableAlias()); + } + private static boolean isExistingRecord(Patch patch) { return StringUtils.isNotBlank(patch.getId()); } From c564eccc10661e8cc3e1e9c39bbfefa433bf3482 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 13 Jan 2022 16:57:14 +0400 Subject: [PATCH 308/405] DDP-7121 | start of fixing if date type field contains empty value --- .../java/org/broadinstitute/dsm/model/elastic/Util.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 2d37cf44c..0c858f9a1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -17,6 +17,7 @@ import org.broadinstitute.dsm.model.elastic.export.parse.DynamicFieldsParser; import org.broadinstitute.dsm.model.elastic.export.parse.ValueParser; import org.broadinstitute.dsm.statics.DBConstants; +import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ObjectMapperSingleton; import org.broadinstitute.dsm.util.ParticipantUtil; import org.broadinstitute.dsm.util.PatchUtil; @@ -137,11 +138,13 @@ static Map convertToMap(String fieldName, Object fieldValue, Str String field = object.getKey(); DYNAMIC_FIELDS_PARSER.setFieldName(field); DYNAMIC_FIELDS_PARSER.setRealm(realm); - Object value = DYNAMIC_FIELDS_PARSER.parse(String.valueOf(object.getValue())); + String elementValue = String.valueOf(object.getValue()); + if (StringUtils.isBlank(elementValue)) continue; + Object parsedValue = DYNAMIC_FIELDS_PARSER.parse(elementValue); String camelCaseField = underscoresToCamelCase(field); - transformedMap.put(camelCaseField, value); + transformedMap.put(camelCaseField, parsedValue); } - finalResult = Map.of("dynamicFields", transformedMap); + finalResult = Map.of(ESObjectConstants.DYNAMIC_FIELDS, transformedMap); break; default: finalResult = new HashMap<>(Map.of(underscoresToCamelCase(fieldName), fieldValue)); From c262cd55faaede2f4340644712f2f9156e5a43d1 Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 13 Jan 2022 17:06:36 +0400 Subject: [PATCH 309/405] DDP-7121 | convert unnapropriate values for date fields as nulls --- src/main/java/org/broadinstitute/dsm/model/elastic/Util.java | 1 - .../dsm/model/elastic/export/parse/ValueParser.java | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 0c858f9a1..8601f98f3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -139,7 +139,6 @@ static Map convertToMap(String fieldName, Object fieldValue, Str DYNAMIC_FIELDS_PARSER.setFieldName(field); DYNAMIC_FIELDS_PARSER.setRealm(realm); String elementValue = String.valueOf(object.getValue()); - if (StringUtils.isBlank(elementValue)) continue; Object parsedValue = DYNAMIC_FIELDS_PARSER.parse(elementValue); String camelCaseField = underscoresToCamelCase(field); transformedMap.put(camelCaseField, parsedValue); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/ValueParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/ValueParser.java index 46d3a2bd0..517ad6f80 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/ValueParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/ValueParser.java @@ -1,5 +1,7 @@ package org.broadinstitute.dsm.model.elastic.export.parse; +import org.apache.commons.lang3.StringUtils; + public class ValueParser extends BaseParser { @Override @@ -12,6 +14,7 @@ protected Object forNumeric(String value) { @Override protected Object forDate(String value) { + if (StringUtils.isBlank(value)) return null; return value; } From baff1b585d8da50bd81be0875e2b0d8d217e3142 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 14 Jan 2022 10:44:37 +0400 Subject: [PATCH 310/405] DDP-7121 | start refactoring messy code of filtering --- .../org/broadinstitute/dsm/db/ViewFilter.java | 5 +- .../BaseFilterParticipantList.java | 48 ++++++++++++------- .../model/participant/ParticipantWrapper.java | 1 + 3 files changed, 37 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/ViewFilter.java b/src/main/java/org/broadinstitute/dsm/db/ViewFilter.java index cb6aa7e53..2ab89f5a7 100644 --- a/src/main/java/org/broadinstitute/dsm/db/ViewFilter.java +++ b/src/main/java/org/broadinstitute/dsm/db/ViewFilter.java @@ -257,7 +257,10 @@ private static Result saveFilter(@NonNull ViewFilter viewFilter, @NonNull String public static void addQueryCondition(@NonNull Map queryConditions, DBElement dbElement, Filter filter) { if (dbElement != null) { String queryCondition = ""; - String propertyName = StringUtils.isNotBlank(filter.getParentName()) ? filter.getParentName() : filter.getParticipantColumn().getTableAlias(); +// String propertyName = StringUtils.isNotBlank(filter.getParentName()) ? filter.getParentName() : filter.getParticipantColumn().getTableAlias(); + String propertyName = StringUtils.isNotBlank(filter.getParticipantColumn().getTableAlias()) + ? filter.getParticipantColumn().getTableAlias() + : filter.getParentName(); if (queryConditions.containsKey(propertyName)) { queryCondition = queryConditions.get(propertyName); } diff --git a/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java b/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java index a65896977..dc55d5244 100644 --- a/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java +++ b/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java @@ -18,6 +18,8 @@ import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.Filter; +import org.broadinstitute.dsm.model.NameValue; +import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; import org.broadinstitute.dsm.model.participant.ParticipantWrapper; import org.broadinstitute.dsm.model.filter.BaseFilter; @@ -70,9 +72,12 @@ protected ParticipantWrapperResult filterParticipantList(Filter[] filters, Map Date: Fri, 14 Jan 2022 11:14:52 +0400 Subject: [PATCH 311/405] DDP-7121 | set parentname to tablealias --- src/main/java/org/broadinstitute/dsm/db/ViewFilter.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/broadinstitute/dsm/db/ViewFilter.java b/src/main/java/org/broadinstitute/dsm/db/ViewFilter.java index 2ab89f5a7..0745d9835 100644 --- a/src/main/java/org/broadinstitute/dsm/db/ViewFilter.java +++ b/src/main/java/org/broadinstitute/dsm/db/ViewFilter.java @@ -271,6 +271,7 @@ public static void addQueryCondition(@NonNull Map queryCondition if (queryConditions.containsKey("ES")) { queryCondition = queryConditions.get("ES"); } + filter.setParentName(filter.getParticipantColumn().getTableAlias()); queryConditions.put("ES", queryCondition.concat(Filter.getQueryStringForFiltering(filter, null))); } } From 4d9187d1ed304a887c4be634ff3544c08ed99283 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 14 Jan 2022 11:49:07 +0400 Subject: [PATCH 312/405] DDP-7121 | fix multiple options filter, add test for it --- .../filter/splitter/MultipleOptionsSplitter.java | 13 +++++++++---- .../splitter/MultipleOptionsSplitterTest.java | 10 ++++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/MultipleOptionsSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/MultipleOptionsSplitter.java index 1cf9975c2..c34d12e0c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/MultipleOptionsSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/MultipleOptionsSplitter.java @@ -1,8 +1,11 @@ package org.broadinstitute.dsm.model.elastic.filter.splitter; +import java.util.List; + import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.Filter; import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.model.elastic.filter.AndOrFilterSeparator; public class MultipleOptionsSplitter extends BaseSplitter { @@ -25,11 +28,13 @@ public String getInnerProperty() { @Override public String[] split() { - String[] multipleFilters = filter + String multipleFilters = + Filter.OR + + filter .replace(Filter.OPEN_PARENTHESIS, StringUtils.EMPTY) .replace(Filter.CLOSE_PARENTHESIS, StringUtils.EMPTY) - .trim() - .split(Filter.OR_TRIMMED); - return multipleFilters; + .trim(); + List splittedFilter = new AndOrFilterSeparator(multipleFilters).parseFiltersByLogicalOperators().get(Filter.OR_TRIMMED); + return splittedFilter.toArray(new String[]{}); } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/MultipleOptionsSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/MultipleOptionsSplitterTest.java index b75fcd13f..515d81143 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/MultipleOptionsSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/MultipleOptionsSplitterTest.java @@ -17,4 +17,14 @@ public void getInnerProperty() { multipleSplitter.setFilter(filter); assertEquals("faxSent", multipleSplitter.getInnerProperty()); } + + @Test + public void contentContainsOR() { + String filter = "( d.status = 'EXITED_BEFORE_ENROLLMENT' OR d.status = 'EXITED_AFTER_ENROLLMENT' )"; + String[] filters = new String[] {"d.status = 'EXITED_BEFORE_ENROLLMENT'", "d.status = 'EXITED_AFTER_ENROLLMENT'"}; + BaseSplitter multipleSplitter = SplitterFactory.createSplitter(Operator.MULTIPLE_OPTIONS, ""); + multipleSplitter.setFilter(filter); + String[] actualFilters = multipleSplitter.split(); + assertArrayEquals(filters, actualFilters); + } } \ No newline at end of file From e7d6fefead0d7529e9de5eba2d01ea7415d71d89 Mon Sep 17 00:00:00 2001 From: Nika Date: Fri, 14 Jan 2022 12:35:47 +0400 Subject: [PATCH 313/405] DDP-7121 | fix search for ddp_participant_data dynamic fields --- .../filter/participant/BaseFilterParticipantList.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java b/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java index dc55d5244..82e783c78 100644 --- a/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java +++ b/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java @@ -27,6 +27,7 @@ import org.broadinstitute.dsm.model.participant.ParticipantWrapperPayload; import org.broadinstitute.dsm.model.participant.ParticipantWrapperResult; import org.broadinstitute.dsm.statics.DBConstants; +import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.broadinstitute.dsm.util.ObjectMapperSingleton; import org.broadinstitute.dsm.util.ParticipantUtil; @@ -89,9 +90,12 @@ protected ParticipantWrapperResult filterParticipantList(Filter[] filters, Map Date: Fri, 14 Jan 2022 16:46:36 +0400 Subject: [PATCH 314/405] DDP-7121 | fixing filtering for dynamic field's options --- .../org/broadinstitute/dsm/model/Filter.java | 61 +++++++++++-------- .../filter/splitter/JsonExtractSplitter.java | 1 + .../BaseFilterParticipantList.java | 17 +++++- 3 files changed, 54 insertions(+), 25 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/Filter.java b/src/main/java/org/broadinstitute/dsm/model/Filter.java index e68d7e370..236b6d8a0 100644 --- a/src/main/java/org/broadinstitute/dsm/model/Filter.java +++ b/src/main/java/org/broadinstitute/dsm/model/Filter.java @@ -6,6 +6,7 @@ import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; +import java.util.Objects; import lombok.Getter; import lombok.NonNull; @@ -199,30 +200,7 @@ else if (filter.isNotEmpty()) { } } else if (ADDITIONAL_VALUES.equals(filter.getType())) { - String jsonExtract = "JSON_EXTRACT ( "; - query = AND + - jsonExtract + filter.getParentName() + DBConstants.ALIAS_DELIMITER + dbElement.getColumnName() + " , '$." + filter.getFilter2().getName() + "' ) "; - if (filter.isEmpty()) { - finalQuery = query + IS_NULL + " "; - } - else if (filter.isNotEmpty()) { - finalQuery = query + IS_NOT_NULL + " "; - } - else { - String notNullQuery = AND + filter.getParentName() + DBConstants.ALIAS_DELIMITER + dbElement.getColumnName() + IS_NOT_NULL; - if (filter.getFilter1() != null && filter.getFilter1().getValue() != null && StringUtils.isNotBlank(String.valueOf(filter.getFilter1().getValue()))) { - query = AND + jsonExtract + filter.getParentName() + DBConstants.ALIAS_DELIMITER + dbElement.getColumnName() + " , '$." + filter.getFilter2().getName() + "' ) "; - if (filter.isExactMatch()) { - query += EQUALS + "'#'"; - query = query.replaceAll("#", String.valueOf(filter.getFilter1().getValue())); - } - else { - query += " " + LIKE + " '%#%'"; - query = query.replaceAll("#", String.valueOf(filter.getFilter1().getValue())); - } - } - finalQuery = notNullQuery + query; - } + finalQuery = buildJsonExtract(filter, dbElement); } else if (JSON_ARRAY.equals(filter.getType())) { query = AND + filter.getParentName() + DBConstants.ALIAS_DELIMITER + dbElement.getColumnName() ; @@ -278,6 +256,41 @@ else if (filter.getFilter1() != null && filter.getFilter2().getValue() != null & return finalQuery; } + private static String buildJsonExtract(Filter filter, DBElement dbElement) { + String query; + String finalQuery; + String jsonExtract = "JSON_EXTRACT ( "; + query = AND + + jsonExtract + filter.getParentName() + DBConstants.ALIAS_DELIMITER + dbElement.getColumnName() + " , '$." + filter.getFilter2().getName() + "' ) "; + if (filter.isEmpty()) { + finalQuery = query + IS_NULL + " "; + } + else if (filter.isNotEmpty()) { + finalQuery = query + IS_NOT_NULL + " "; + } + else { + String notNullQuery = AND + filter.getParentName() + DBConstants.ALIAS_DELIMITER + dbElement.getColumnName() + IS_NOT_NULL; + if (filter.getFilter1() != null && filter.getFilter1().getValue() != null && StringUtils.isNotBlank(String.valueOf( + filter.getFilter1().getValue()))) { + query = AND + jsonExtract + filter.getParentName() + DBConstants.ALIAS_DELIMITER + dbElement.getColumnName() + " , '$." + filter.getFilter2().getName() + "' ) "; + if (filter.isExactMatch()) { + query += EQUALS + "'#'"; + query = query.replaceAll("#", String.valueOf(filter.getFilter1().getValue())); + } + else { + query += " " + LIKE + " '%#%'"; + query = query.replaceAll("#", String.valueOf(filter.getFilter1().getValue())); + } + } + if (Objects.nonNull(filter.getSelectedOptions())) { + finalQuery = query; + } else { + finalQuery = notNullQuery + query; + } + } + return finalQuery; + } + /** * Check if filter value is not null and is not blank * @return boolean is true if a filter is not null and not blank diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonExtractSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonExtractSplitter.java index 1ccf1eadd..a1614d337 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonExtractSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonExtractSplitter.java @@ -11,6 +11,7 @@ public class JsonExtractSplitter extends BaseSplitter { private BaseSplitter decoratedSplitter; + public JsonExtractSplitter(BaseSplitter splitter) { this.decoratedSplitter = splitter; } diff --git a/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java b/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java index 82e783c78..44754bcc6 100644 --- a/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java +++ b/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java @@ -4,6 +4,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.regex.Pattern; import com.google.gson.Gson; @@ -20,6 +21,7 @@ import org.broadinstitute.dsm.model.Filter; import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.model.elastic.filter.AndOrFilterSeparator; import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; import org.broadinstitute.dsm.model.participant.ParticipantWrapper; import org.broadinstitute.dsm.model.filter.BaseFilter; @@ -95,7 +97,20 @@ protected ParticipantWrapperResult filterParticipantList(Filter[] filters, Map String.join(Filter.SPACE, prev, curr)); + } + } else { + queryConditions.put(DBConstants.DDP_PARTICIPANT_DATA_ALIAS, Filter.getQueryStringForFiltering(filter, dbElement)); + queryConditions.merge(DBConstants.DDP_PARTICIPANT_DATA_ALIAS, Filter.getQueryStringForFiltering(filter, + dbElement), (prev, curr) -> String.join(Filter.SPACE, prev, curr)); + } } else { dbElement = columnNameMap.get(tableAlias + "." + tmpName); ViewFilter.addQueryCondition(queryConditions, dbElement, filter); From beb4d6e76144a9ddf68cc4ea7413019e56cf69fa Mon Sep 17 00:00:00 2001 From: Nika Date: Fri, 14 Jan 2022 17:17:16 +0400 Subject: [PATCH 315/405] DDP-7121 | change if condition for selecting multi options enum Operator --- .../org/broadinstitute/dsm/model/elastic/filter/Operator.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java index 5ee9b3ba3..d42de6536 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java @@ -38,7 +38,7 @@ public static Operator extract(String filter) { filter = filter.trim(); if (filter.endsWith(Filter.IS_NOT_NULL_TRIMMED) && !filter.startsWith(Filter.JSON_EXTRACT)) return IS_NOT_NULL; - else if (filter.startsWith(Filter.OPEN_PARENTHESIS)) + else if (filter.startsWith(Filter.OPEN_PARENTHESIS) && filter.endsWith(Filter.CLOSE_PARENTHESIS)) return MULTIPLE_OPTIONS; else if (filter.startsWith(Filter.DATE_FORMAT)) return STR_DATE; @@ -52,6 +52,7 @@ else if (filter.contains(Filter.JSON_EXTRACT)) return JSON_EXTRACT; String operator = Arrays.stream(filter.split(Filter.SPACE)) .filter(StringUtils::isNotBlank) + .filter(str -> Arrays.asList(Operator.values()).contains(str)) .filter(str -> Arrays.stream(Operator.values()).anyMatch(op -> op.value.equals(str))) .findFirst() .orElse(StringUtils.EMPTY); From 02e7ccf33001e2ec0e84193dd1135590b4d3796d Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 14 Jan 2022 17:47:01 +0400 Subject: [PATCH 316/405] DDP-7121 | change operator extracting --- .../dsm/model/elastic/filter/Operator.java | 25 +++++++------------ .../filter/splitter/SplitterFactory.java | 8 +++--- .../model/elastic/filter/OperatorTest.java | 7 ++++++ 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java index d42de6536..2fcbf6dd0 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java @@ -35,27 +35,20 @@ public static Operator getOperator(String value) { } public static Operator extract(String filter) { - filter = filter.trim(); - if (filter.endsWith(Filter.IS_NOT_NULL_TRIMMED) && !filter.startsWith(Filter.JSON_EXTRACT)) - return IS_NOT_NULL; - else if (filter.startsWith(Filter.OPEN_PARENTHESIS) && filter.endsWith(Filter.CLOSE_PARENTHESIS)) + String[] splittedFilter = filter.split(Filter.SPACE); + if (isMultipleOptions(splittedFilter)) return MULTIPLE_OPTIONS; - else if (filter.startsWith(Filter.DATE_FORMAT)) - return STR_DATE; - else if (filter.startsWith(Filter.DATE)) - return DATE; - else if (filter.contains(Filter.DATE_GREATER)) - return DATE_GREATER; - else if (filter.contains(Filter.DATE_LESS)) - return DATE_LESS; - else if (filter.contains(Filter.JSON_EXTRACT)) - return JSON_EXTRACT; - String operator = Arrays.stream(filter.split(Filter.SPACE)) + String operator = Arrays.stream(splittedFilter) .filter(StringUtils::isNotBlank) - .filter(str -> Arrays.asList(Operator.values()).contains(str)) + .filter(str -> !Filter.OPEN_PARENTHESIS.equals(str) && !Filter.CLOSE_PARENTHESIS.equals(str)) .filter(str -> Arrays.stream(Operator.values()).anyMatch(op -> op.value.equals(str))) .findFirst() .orElse(StringUtils.EMPTY); return getOperator(operator); } + + private static boolean isMultipleOptions(String[] splittedFilter) { + return splittedFilter.length > 0 && Filter.OPEN_PARENTHESIS.equals(splittedFilter[0]) && + Filter.CLOSE_PARENTHESIS.equals(splittedFilter[splittedFilter.length - 1]); + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/SplitterFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/SplitterFactory.java index ae9dc9000..193bc6925 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/SplitterFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/SplitterFactory.java @@ -44,10 +44,10 @@ public static BaseSplitter createSplitter(Operator operator, String filterValue) break; case JSON_EXTRACT: Operator decoratedOperator = Operator.extract(filterValue.replace(Filter.JSON_EXTRACT, StringUtils.EMPTY)); - if (Operator.IS_NOT_NULL.compareTo(decoratedOperator) != 0) { - splitter = new JsonExtractSplitter(); - break; - } +// if (Operator.IS_NOT_NULL.compareTo(decoratedOperator) != 0) { +// splitter = new JsonExtractSplitter(); +// break; +// } BaseSplitter decoratedSplitter = createSplitter(decoratedOperator, filterValue); splitter = new JsonExtractSplitter(decoratedSplitter); break; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java index fee780afe..2506bb212 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java @@ -41,4 +41,11 @@ public void extractLogicalOperator() { assertEquals(Operator.LIKE, likeOperator); assertEquals(Operator.DIAMOND_EQUALS, diamondsOperator); } + + @Test + public void extractJsonExtractValue() { + String filter = " ( d.additional_values_json , '$.status' ) LIKE '%EXITED_BEFORE_ENROLLMENT%'"; + Operator likeOperator = Operator.extract(filter); + assertEquals(Operator.LIKE, likeOperator); + } } \ No newline at end of file From c7d7292b892f5b2fd04f44cf9a024250b06e05a9 Mon Sep 17 00:00:00 2001 From: Nika Date: Fri, 14 Jan 2022 18:16:02 +0400 Subject: [PATCH 317/405] DDP-7121 | implement correct splitting mechanism for LikeSplitter --- .../elastic/filter/splitter/LikeSplitter.java | 9 ++++++++- .../filter/splitter/LikeSplitterTest.java | 19 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/LikeSplitterTest.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/LikeSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/LikeSplitter.java index b27e02515..92046432e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/LikeSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/LikeSplitter.java @@ -3,10 +3,17 @@ import org.broadinstitute.dsm.model.Filter; +import java.util.Arrays; +import java.util.stream.Collectors; + public class LikeSplitter extends BaseSplitter { @Override public String[] split() { - return filter.split(Filter.LIKE_TRIMMED); + String[] splittedFilter = filter.split(Filter.LIKE_TRIMMED); + return Arrays.stream(splittedFilter) + .map(str -> str.replace("%","")) + .collect(Collectors.toList()) + .toArray(new String[] {}); } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/LikeSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/LikeSplitterTest.java new file mode 100644 index 000000000..5fb5fd621 --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/LikeSplitterTest.java @@ -0,0 +1,19 @@ +package org.broadinstitute.dsm.model.elastic.filter.splitter; + +import org.broadinstitute.dsm.model.elastic.filter.Operator; +import org.junit.Assert; +import org.junit.Test; + +import static org.junit.Assert.*; + +public class LikeSplitterTest { + + @Test + public void split() { + String filter = "JSON_EXTRACT ( d.additional_values_json , '$.status' ) LIKE '%EXITED_BEFORE_ENROLLMENT%'"; + BaseSplitter splitter = SplitterFactory.createSplitter(Operator.JSON_EXTRACT, filter); + splitter.setFilter(filter); + String actual = splitter.getValue()[0]; + Assert.assertEquals("'EXITED_BEFORE_ENROLLMENT'", actual); + } +} \ No newline at end of file From a8bea5262ab3eb4ab7ef0697c05d49c99320a0e6 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 14 Jan 2022 19:16:04 +0400 Subject: [PATCH 318/405] DDP-7121 | removed dead code --- .../broadinstitute/dsm/db/MedicalRecord.java | 4 +- .../dsm/db/OncHistoryDetail.java | 5 +- .../broadinstitute/dsm/db/Participant.java | 4 +- .../ddp/participant/ParticipantRecordDto.java | 7 +- .../BaseFilterParticipantList.java | 210 ++---------------- .../dsm/statics/DBConstants.java | 2 +- .../org/broadinstitute/dsm/FilterTest.java | 201 ----------------- 7 files changed, 30 insertions(+), 403 deletions(-) delete mode 100644 src/test/java/org/broadinstitute/dsm/FilterTest.java diff --git a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java index 066585067..bfef4f9ff 100644 --- a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java +++ b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java @@ -339,7 +339,7 @@ public boolean isFollowupRequired() { alias = DBConstants.DDP_MEDICAL_RECORD_ALIAS, primaryKey = DBConstants.MEDICAL_RECORD_ID, columnPrefix = "") - @ColumnName (DBConstants.ADDITIONAL_VALUES) + @ColumnName (DBConstants.ADDITIONAL_VALUES_JSON) @JsonProperty("dynamicFields") @SerializedName("dynamicFields") private String additionalValuesJson; @@ -453,7 +453,7 @@ public static MedicalRecord getMedicalRecord(@NonNull ResultSet rs) throws SQLEx new Gson().fromJson(rs.getString(DBConstants.FOLLOW_UP_REQUESTS), FollowUp[].class), rs.getBoolean(DBConstants.FOLLOWUP_REQUIRED), rs.getString(DBConstants.FOLLOWUP_REQUIRED_TEXT), - rs.getString(DBConstants.ADDITIONAL_VALUES), + rs.getString(DBConstants.ADDITIONAL_VALUES_JSON), rs.getString(DBConstants.MR_UNABLE_OBTAIN_TEXT), rs.getString(DBConstants.DDP_PARTICIPANT_ID) ); diff --git a/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java b/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java index 982d4ce44..6f55eebc1 100644 --- a/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java +++ b/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java @@ -4,7 +4,6 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.annotations.SerializedName; import lombok.Data; import lombok.NonNull; @@ -166,7 +165,7 @@ public class OncHistoryDetail { @ColumnName (DBConstants.GENDER) private String gender; - @ColumnName (DBConstants.ADDITIONAL_VALUES) + @ColumnName (DBConstants.ADDITIONAL_VALUES_JSON) @JsonProperty("dynamicFields") @SerializedName("dynamicFields") private String additionalValuesJson; @@ -299,7 +298,7 @@ public static OncHistoryDetail getOncHistoryDetail(@NonNull ResultSet rs) throws rs.getString(DBConstants.DDP_ONC_HISTORY_DETAIL_ALIAS + DBConstants.ALIAS_DELIMITER + DBConstants.FAX_CONFIRMED_3), rs.getString(DBConstants.TISSUE_RECEIVED), rs.getString(DBConstants.GENDER), - rs.getString(DBConstants.DDP_ONC_HISTORY_DETAIL_ALIAS + DBConstants.ALIAS_DELIMITER + DBConstants.ADDITIONAL_VALUES), tissues, + rs.getString(DBConstants.DDP_ONC_HISTORY_DETAIL_ALIAS + DBConstants.ALIAS_DELIMITER + DBConstants.ADDITIONAL_VALUES_JSON), tissues, rs.getString(DBConstants.TISSUE_PROBLEM_OPTION), rs.getString(DBConstants.DESTRUCTION_POLICY), rs.getBoolean(DBConstants.UNABLE_OBTAIN_TISSUE) diff --git a/src/main/java/org/broadinstitute/dsm/db/Participant.java b/src/main/java/org/broadinstitute/dsm/db/Participant.java index f7c86109b..0aa335630 100644 --- a/src/main/java/org/broadinstitute/dsm/db/Participant.java +++ b/src/main/java/org/broadinstitute/dsm/db/Participant.java @@ -130,7 +130,7 @@ public class Participant { alias = DBConstants.DDP_PARTICIPANT_RECORD_ALIAS, primaryKey = DBConstants.PARTICIPANT_ID, columnPrefix = "") - @ColumnName (DBConstants.ADDITIONAL_VALUES) + @ColumnName (DBConstants.ADDITIONAL_VALUES_JSON) @JsonProperty("dynamicFields") @SerializedName("dynamicFields") private String additionalValuesJson; @@ -197,7 +197,7 @@ public static Participant getParticipant(@NonNull Map assignee rs.getString(DBConstants.DDP_PARTICIPANT_RECORD_ALIAS + DBConstants.ALIAS_DELIMITER + DBConstants.NOTES), rs.getBoolean(DBConstants.DDP_PARTICIPANT_RECORD_ALIAS + DBConstants.ALIAS_DELIMITER + DBConstants.MINIMAL_MR), rs.getBoolean(DBConstants.DDP_PARTICIPANT_RECORD_ALIAS + DBConstants.ALIAS_DELIMITER + DBConstants.ABSTRACTION_READY), - rs.getString(DBConstants.DDP_PARTICIPANT_RECORD_ALIAS + DBConstants.ALIAS_DELIMITER + DBConstants.ADDITIONAL_VALUES), + rs.getString(DBConstants.DDP_PARTICIPANT_RECORD_ALIAS + DBConstants.ALIAS_DELIMITER + DBConstants.ADDITIONAL_VALUES_JSON), rs.getLong(DBConstants.EXIT_DATE)); return participant; } diff --git a/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantRecordDto.java b/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantRecordDto.java index 203e24399..b65613f62 100644 --- a/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantRecordDto.java +++ b/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantRecordDto.java @@ -8,12 +8,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.gson.annotations.SerializedName; -import lombok.Data; -import lombok.Getter; import org.broadinstitute.dsm.db.structure.ColumnName; -import org.broadinstitute.dsm.db.structure.TableName; import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.util.ObjectMapperSingleton; @@ -42,7 +37,7 @@ public class ParticipantRecordDto { @ColumnName (DBConstants.ABSTRACTION_READY) private Integer abstractionReady; - @ColumnName (DBConstants.ADDITIONAL_VALUES) + @ColumnName (DBConstants.ADDITIONAL_VALUES_JSON) @JsonProperty("dynamicFields") private String additionalValuesJson; diff --git a/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java b/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java index 44754bcc6..d5264dbe2 100644 --- a/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java +++ b/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java @@ -1,8 +1,6 @@ package org.broadinstitute.dsm.model.filter.participant; -import java.time.LocalDate; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Objects; import java.util.regex.Pattern; @@ -10,29 +8,24 @@ import com.google.gson.Gson; import lombok.NonNull; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.math.NumberUtils; import org.broadinstitute.dsm.db.DDPInstance; import org.broadinstitute.dsm.db.ViewFilter; import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; -import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.Filter; import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.filter.AndOrFilterSeparator; import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; -import org.broadinstitute.dsm.model.participant.ParticipantWrapper; import org.broadinstitute.dsm.model.filter.BaseFilter; import org.broadinstitute.dsm.model.filter.Filterable; +import org.broadinstitute.dsm.model.participant.ParticipantWrapper; import org.broadinstitute.dsm.model.participant.ParticipantWrapperPayload; import org.broadinstitute.dsm.model.participant.ParticipantWrapperResult; import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.statics.ESObjectConstants; -import org.broadinstitute.dsm.util.ElasticSearchUtil; -import org.broadinstitute.dsm.util.ObjectMapperSingleton; -import org.broadinstitute.dsm.util.ParticipantUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,16 +34,7 @@ public abstract class BaseFilterParticipantList extends BaseFilter implements Fi private static final Logger logger = LoggerFactory.getLogger(BaseFilterParticipantList.class); public static final String PARTICIPANT_DATA = "participantData"; public static final String OPTIONS = "OPTIONS"; - public static final String RADIO = "RADIO"; - public static final String PARTICIPANTS = "PARTICIPANTS"; protected static final Gson GSON = new Gson(); - private static final Pattern DATE_PATTERN = Pattern.compile( - "^\\d{4}-\\d{2}-\\d{2}$"); - private final ParticipantDataDao participantDataDao; - - { - participantDataDao = new ParticipantDataDao(); - } public BaseFilterParticipantList() { super(null); @@ -63,7 +47,6 @@ public BaseFilterParticipantList(String jsonBody) { protected ParticipantWrapperResult filterParticipantList(Filter[] filters, Map columnNameMap, @NonNull DDPInstance instance) { Map queryConditions = new HashMap<>(); - List allParticipantData = null; DDPInstanceDto ddpInstanceDto = new DDPInstanceDao().getDDPInstanceByInstanceName(realm).orElseThrow(); ParticipantWrapperPayload.Builder participantWrapperPayload = new ParticipantWrapperPayload.Builder() .withDdpInstanceDto(ddpInstanceDto) @@ -71,13 +54,10 @@ protected ParticipantWrapperResult filterParticipantList(Filter[] filters, Map allIdsForParticipantDataFiltering = new HashMap<>(); - int numberOfParticipantDataFilters = 0; for (Filter filter : filters) { if (filter != null) { String tableAlias = null; if (filter.getParticipantColumn() != null) { -// tableAlias = StringUtils.isNotBlank(filter.getParentName()) ? filter.getParentName() : filter.getParticipantColumn().getTableAlias(); tableAlias = StringUtils.isNotBlank(filter.getParticipantColumn().getTableAlias()) ? filter.getParticipantColumn().getTableAlias() : filter.getParentName(); @@ -90,53 +70,15 @@ protected ParticipantWrapperResult filterParticipantList(Filter[] filters, Map String.join(Filter.SPACE, prev, curr)); - } - } else { - queryConditions.put(DBConstants.DDP_PARTICIPANT_DATA_ALIAS, Filter.getQueryStringForFiltering(filter, dbElement)); - queryConditions.merge(DBConstants.DDP_PARTICIPANT_DATA_ALIAS, Filter.getQueryStringForFiltering(filter, - dbElement), (prev, curr) -> String.join(Filter.SPACE, prev, curr)); - } + if (PARTICIPANT_DATA.equals(filter.getParticipantColumn().getTableAlias())) { + addParticipantDataQueryToQueryConditions(queryConditions, filter, tmpName); } else { dbElement = columnNameMap.get(tableAlias + "." + tmpName); ViewFilter.addQueryCondition(queryConditions, dbElement, filter); } } -// if (filter.getParticipantColumn() != null && (PARTICIPANT_DATA.equals(filter.getParticipantColumn().tableAlias))) { -// -// -// if (allParticipantData == null) { -// allParticipantData = participantDataDao -// .getParticipantDataByInstanceId(Integer.parseInt(instance.getDdpInstanceId())); -// } -// numberOfParticipantDataFilters++; -// addParticipantDataIdsForFilters(filter, tmpName, allParticipantData, allIdsForParticipantDataFiltering); -// } -// else { -// if (StringUtils.isNotBlank(tmpName)) { -// dbElement = columnNameMap.get(tableAlias + "." + tmpName); -// } -// ViewFilter.addQueryCondition(queryConditions, dbElement, filter); -// } } } - if (numberOfParticipantDataFilters != 0) { - addParticipantDataConditionsToQuery(allIdsForParticipantDataFiltering, queryConditions, numberOfParticipantDataFilters); - } } if (!queryConditions.isEmpty()) { @@ -157,135 +99,27 @@ protected ParticipantWrapperResult filterParticipantList(Filter[] filters, Map allIdsForParticipantDataFiltering, Map queryConditions, int filtersLength) { - String newCondition = createNewConditionByIds(allIdsForParticipantDataFiltering, filtersLength); - queryConditions.merge(ElasticSearchUtil.ES, newCondition, (prev, next) -> prev + next); - } - - public String createNewConditionByIds(Map allIdsForParticipantDataFiltering, int filtersLength) { - StringBuilder newCondition = new StringBuilder(ElasticSearchUtil.AND); - int i = 0; - for (Map.Entry entry: allIdsForParticipantDataFiltering.entrySet()) { - if (entry.getValue() != filtersLength) { - continue; - } - if (i == 0) { - newCondition.append(ParticipantUtil.isGuid(entry.getKey()) ? ElasticSearchUtil.BY_PROFILE_GUID + entry.getKey() : ElasticSearchUtil.BY_PROFILE_LEGACY_ALTPID + entry.getKey()); - } else { - newCondition.append(ParticipantUtil.isGuid(entry.getKey()) ? ElasticSearchUtil.BY_GUIDS + entry.getKey() : ElasticSearchUtil.BY_LEGACY_ALTPIDS + entry.getKey()); - } - i++; - } - if (i == 0) { - newCondition.append(ElasticSearchUtil.BY_PROFILE_GUID + ElasticSearchUtil.EMPTY); - } - newCondition.append(ElasticSearchUtil.CLOSING_PARENTHESIS); - return newCondition.toString(); - } - - public void addParticipantDataIdsForFilters(Filter filter, String fieldName, List allParticipantData, - Map allIdsForParticipantDataFiltering) { - Map participantIdsForQuery = new HashMap(); - Map participantsNotToAdd = new HashMap(); - for (ParticipantDataDto participantData : allParticipantData) { - String data = participantData.getData().orElse(null); - String fieldTypeId = participantData.getFieldTypeId().orElse(null); - if (data == null || fieldTypeId == null) { - continue; - } - String ddpParticipantId = participantData.getDdpParticipantId().orElse(null); - Map dataMap = ObjectMapperSingleton.readValue(data, Map.class); - boolean questionWithOptions = (OPTIONS.equals(filter.getType()) || RADIO.equals(filter.getType())) && filter.getSelectedOptions() != null; - boolean notEmptyCheck = filter.isNotEmpty() && dataMap.get(fieldName) != null && !dataMap.get(fieldName).isEmpty(); - boolean emptyCheck = filter.isEmpty() && (dataMap.get(fieldName) == null || dataMap.get(fieldName).isEmpty()); - if (notEmptyCheck || emptyCheck && !participantsNotToAdd.containsKey(ddpParticipantId)) { - participantIdsForQuery.put(ddpParticipantId, fieldName); - continue; - } - //For the participants, which are saved in several rows (for example AT participants) - boolean shouldNotHaveBeenAdded = filter.isEmpty() && !(dataMap.get(fieldName) == null || dataMap.get(fieldName).isEmpty()) - && !fieldTypeId.contains(PARTICIPANTS); - if (shouldNotHaveBeenAdded) { - participantIdsForQuery.remove(ddpParticipantId); - participantsNotToAdd.put(ddpParticipantId, fieldName); - continue; - } - - if (questionWithOptions) { - for (String option : filter.getSelectedOptions()) { - if (dataMap.get(fieldName) != null && dataMap.get(fieldName) - .equals(option)) { - participantIdsForQuery.put(ddpParticipantId, fieldName); - break; - } - } - } else if (filter.getFilter1() != null && filter.getFilter1().getValue() != null) { - boolean singleValueMatches; - if (filter.isExactMatch()) { - singleValueMatches = dataMap.get(fieldName) != null && dataMap.get(fieldName) - .equals(filter.getFilter1().getValue()); - } else { - singleValueMatches = dataMap.get(fieldName) != null && dataMap.get(fieldName).toLowerCase() - .contains(filter.getFilter1().getValue().toString().toLowerCase()); - } - if (singleValueMatches) { - participantIdsForQuery.put(ddpParticipantId, fieldName); - } else if (filter.getFilter2() != null) { - addConditionForRange(filter, fieldName, dataMap, participantIdsForQuery, ddpParticipantId); - } - } - } - participantIdsForQuery.forEach((key, value) -> allIdsForParticipantDataFiltering - .put(key, allIdsForParticipantDataFiltering.getOrDefault(key, 0) + 1)); - } - - private StringBuilder getNewCondition(Map participantIdsForQuery) { - StringBuilder newCondition = new StringBuilder(ElasticSearchUtil.AND); - int i = 0; - for (String id : participantIdsForQuery.keySet()) { - if (i == 0) { - newCondition.append(ParticipantUtil.isGuid(id) ? ElasticSearchUtil.BY_PROFILE_GUID + id : ElasticSearchUtil.BY_PROFILE_LEGACY_ALTPID + id); - } else { - newCondition.append(ParticipantUtil.isGuid(id) ? ElasticSearchUtil.BY_GUIDS + id : ElasticSearchUtil.BY_LEGACY_ALTPIDS + id); + private void addParticipantDataQueryToQueryConditions(Map queryConditions, Filter filter, String tmpName) { + filter.setFilter1(new NameValue(ESObjectConstants.ADDITIONAL_VALUES_JSON, filter.getFilter1().getValue())); + filter.setFilter2(new NameValue(Util.underscoresToCamelCase(tmpName), null)); + filter.setParentName(DBConstants.DDP_PARTICIPANT_DATA_ALIAS); + filter.setType(Filter.ADDITIONAL_VALUES); + DBElement dbElement = new DBElement(DBConstants.DDP_PARTICIPANT_DATA, DBConstants.DDP_PARTICIPANT_DATA_ALIAS, null, + DBConstants.ADDITIONAL_VALUES_JSON); + if (Objects.nonNull(filter.getSelectedOptions())) { + for (String selectedOption : filter.getSelectedOptions()) { + filter.getFilter1().setValue(selectedOption); + filter.getFilter2().setName(Util.underscoresToCamelCase(tmpName)); + String filterQuery = Filter.OR_TRIMMED +Filter.getQueryStringForFiltering(filter, + dbElement).trim().substring(AndOrFilterSeparator.MINIMUM_STEP_FROM_OPERATOR); + queryConditions.merge(DBConstants.DDP_PARTICIPANT_DATA_ALIAS, filterQuery, + (prev, curr) -> String.join(Filter.SPACE, prev, curr)); } - i++; - } - return newCondition; - } - - private void addConditionForRange(Filter filter, String fieldName, Map dataMap, Map participantIdsForQuery, String ddpParticipantId) { - Object rangeValue1 = filter.getFilter1().getValue(); - Object rangeValue2 = filter.getFilter2().getValue(); - if (rangeValue1 == null || rangeValue2 == null) { - return; - } - - boolean inNumberRange = isInNumberRange(fieldName, dataMap, rangeValue1, rangeValue2); - - boolean inDateRange = isInDateRange(fieldName, dataMap, rangeValue1, rangeValue2); - - if (inNumberRange || inDateRange) { - participantIdsForQuery.put(ddpParticipantId, fieldName); + } else { + queryConditions.put(DBConstants.DDP_PARTICIPANT_DATA_ALIAS, Filter.getQueryStringForFiltering(filter, dbElement)); + queryConditions.merge(DBConstants.DDP_PARTICIPANT_DATA_ALIAS, Filter.getQueryStringForFiltering(filter, + dbElement), (prev, curr) -> String.join(Filter.SPACE, prev, curr)); } } - private boolean isInDateRange(String fieldName, Map dataMap, Object rangeValue1, Object rangeValue2) { - return (StringUtils.isNotBlank(dataMap.get(fieldName))) - && rangeValue1 instanceof String && rangeValue2 instanceof String - && DATE_PATTERN.matcher((String) rangeValue1).matches() && DATE_PATTERN.matcher((String) rangeValue2).matches() - && LocalDate.parse(dataMap.get(fieldName)).compareTo(LocalDate.parse((String) rangeValue1)) >= 0 - && ((LocalDate.parse(dataMap.get(fieldName)).compareTo(LocalDate.parse((String) rangeValue1)) >= 0 - && LocalDate.parse(dataMap.get(fieldName)).compareTo(LocalDate.parse((String) rangeValue2)) < 0) - || (LocalDate.parse(dataMap.get(fieldName)).compareTo(LocalDate.parse((String) rangeValue2)) >= 0 - && LocalDate.parse(dataMap.get(fieldName)).compareTo(LocalDate.parse((String) rangeValue1)) < 0)); - } - - private boolean isInNumberRange(String fieldName, Map dataMap, Object rangeValue1, Object rangeValue2) { - boolean dataIsNumber = dataMap.get(fieldName) != null && NumberUtils.isNumber(dataMap.get(fieldName)); - boolean moreThanFirstNumber = dataIsNumber && rangeValue1 instanceof Double && Double.compare(Double.parseDouble(dataMap.get(fieldName)), (Double) rangeValue1) >= 0; - boolean moreThanSecondNumber = dataIsNumber && rangeValue2 instanceof Double && Double.compare(Double.parseDouble(dataMap.get(fieldName)), (Double) rangeValue2) >= 0; - //range will be starting from the lower number up until the higher number - return (moreThanFirstNumber && !moreThanSecondNumber) || (moreThanSecondNumber && !moreThanFirstNumber); - } - } diff --git a/src/main/java/org/broadinstitute/dsm/statics/DBConstants.java b/src/main/java/org/broadinstitute/dsm/statics/DBConstants.java index d3eb99daa..dd5e8da4f 100644 --- a/src/main/java/org/broadinstitute/dsm/statics/DBConstants.java +++ b/src/main/java/org/broadinstitute/dsm/statics/DBConstants.java @@ -210,7 +210,7 @@ public class DBConstants { public static final String ACCESSION_NUMBER = "accession_number"; public static final String FACILITY = "facility"; public static final String REQUEST = "request"; - public static final String ADDITIONAL_VALUES = "additional_values_json"; + public static final String ADDITIONAL_VALUES_JSON = "additional_values_json"; public static final String COLUMN_NAME = "column_name"; public static final String COLUMN_DISPLAY = "column_display"; public static final String TISSUE_RECEIVED = "tissue_received"; diff --git a/src/test/java/org/broadinstitute/dsm/FilterTest.java b/src/test/java/org/broadinstitute/dsm/FilterTest.java deleted file mode 100644 index 0136cc051..000000000 --- a/src/test/java/org/broadinstitute/dsm/FilterTest.java +++ /dev/null @@ -1,201 +0,0 @@ -package org.broadinstitute.dsm; - -import com.google.gson.Gson; -import org.broadinstitute.dsm.db.dto.user.UserDto; -import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; -import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; -import org.broadinstitute.dsm.db.dao.user.UserDao; -import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; -import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; -import org.broadinstitute.dsm.model.Filter; -import org.broadinstitute.dsm.model.NameValue; -import org.broadinstitute.dsm.model.filter.participant.EmptyFilterParticipantList; -import org.broadinstitute.dsm.util.DBTestUtil; -import org.broadinstitute.dsm.util.ElasticSearchUtil; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.broadinstitute.dsm.TestHelper.setupDB; - -public class FilterTest { - private static final String FILTER_TEST = "FILTER_TEST"; - private static Gson gson; - - private static final String participantId = "RBMJW6ZIXVXBMXUX6M3Q"; - private static final String participantId1 = "RBMJW6ZIXVXBMXUX6M31"; - - private static UserDto userDto; - - private static int participantDataId; - - private static final Map participantData = new HashMap<>(); - - private static int participantDataId1; - - private static final Map participantData1 = new HashMap<>(); - - private static DDPInstanceDto ddpInstanceDto; - private static final DDPInstanceDao ddpInstanceDao = new DDPInstanceDao(); - - private static final UserDao userDao = new UserDao(); - - private static final ParticipantDataDao participantDataDao = new ParticipantDataDao(); - - private static final EmptyFilterParticipantList emptyFilterParticipantList = new EmptyFilterParticipantList(); - - @BeforeClass - public static void doFirst() { - setupDB(); - gson = new Gson(); - - ddpInstanceDto = DBTestUtil.createTestDdpInstance(ddpInstanceDto, ddpInstanceDao, "FilterTestInstance"); - - userDto = DBTestUtil.createTestDsmUser("testFilterUser", "testFilter@family.com", userDao, userDto); - - createDataForParticipant(); - - createParticipantData(); - - } - - private static void createDataForParticipant() { - participantData.put("PARTICIPANT_DEATH_DATE", "2040-10-30"); - participantData1.put("PARTICIPANT_DEATH_DATE", "2040-10-30"); - } - - private static void createParticipantData() { - ParticipantDataDto participantDataDto = - new ParticipantDataDto.Builder() - .withDdpParticipantId(participantId) - .withDdpInstanceId(ddpInstanceDto.getDdpInstanceId()) - .withFieldTypeId(FILTER_TEST) - .withData(gson.toJson(participantData)) - .withLastChanged(System.currentTimeMillis()) - .withChangedBy(userDto.getEmail().orElse("")) - .build(); - participantDataId = participantDataDao.create(participantDataDto); - - ParticipantDataDto participantDataDto1 = - new ParticipantDataDto.Builder() - .withDdpParticipantId(participantId1) - .withDdpInstanceId(ddpInstanceDto.getDdpInstanceId()) - .withFieldTypeId(FILTER_TEST) - .withData(gson.toJson(participantData1)) - .withLastChanged(System.currentTimeMillis()) - .withChangedBy(userDto.getEmail().orElse("")) - .build(); - participantDataId1 = participantDataDao.create(participantDataDto1); - } - - @Test - public void testAddOneParticipantDataFilter() { - Map queryConditions = new HashMap<>(); - Map allParticipantDataForFiltering = new HashMap<>(); - Filter filter = new Filter(false, true, false, false, "DATE", "participantData", - new NameValue("PARTICIPANT_DEATH_DATE", "2040-10-30"), null, null, null); - List allParticipantData = participantDataDao - .getParticipantDataByInstanceId(Integer.parseInt(String.valueOf(ddpInstanceDto.getDdpInstanceId()))); - emptyFilterParticipantList.addParticipantDataIdsForFilters(filter, filter.getFilter1().getName(), allParticipantData, allParticipantDataForFiltering); - emptyFilterParticipantList.addParticipantDataConditionsToQuery(allParticipantDataForFiltering, queryConditions, 1); - Assert.assertEquals(" AND (profile.guid = " + participantId + " OR profile.guid = " + participantId1 + ")", - queryConditions.get(ElasticSearchUtil.ES)); - } - - @Test - public void testAddOneParticipantDataFilterNotEmpty() { - Map queryConditions = new HashMap<>(); - Map allParticipantDataForFiltering = new HashMap<>(); - Filter filter = new Filter(false, true, false, true, "DATE", "participantData", - new NameValue("PARTICIPANT_DEATH_DATE", "2040-10-35"), null, null, null); - List allParticipantData = participantDataDao - .getParticipantDataByInstanceId(Integer.parseInt(String.valueOf(ddpInstanceDto.getDdpInstanceId()))); - emptyFilterParticipantList.addParticipantDataIdsForFilters(filter, filter.getFilter1().getName(), allParticipantData, allParticipantDataForFiltering); - emptyFilterParticipantList.addParticipantDataConditionsToQuery(allParticipantDataForFiltering, queryConditions, 1); - Assert.assertEquals(" AND (profile.guid = " + participantId + " OR profile.guid = " + participantId1 + ")", - queryConditions.get(ElasticSearchUtil.ES)); - } - - @Test - public void testAddOneParticipantDataFilterEmpty() { - Map queryConditions = new HashMap<>(); - Map allParticipantDataForFiltering = new HashMap<>(); - Filter filter = new Filter(false, true, true, false, "DATE", "participantData", - new NameValue("PARTICIPANT_NONEXISTENT_DATE", "2040-10-35"), null, null, null); - List allParticipantData = participantDataDao - .getParticipantDataByInstanceId(Integer.parseInt(String.valueOf(ddpInstanceDto.getDdpInstanceId()))); - emptyFilterParticipantList.addParticipantDataIdsForFilters(filter, filter.getFilter1().getName(), allParticipantData, allParticipantDataForFiltering); - emptyFilterParticipantList.addParticipantDataConditionsToQuery(allParticipantDataForFiltering, queryConditions, 1); - Assert.assertEquals(" AND (profile.guid = " + participantId + " OR profile.guid = " + participantId1 + ")", - queryConditions.get(ElasticSearchUtil.ES)); - } - - @Test - public void testAddOneParticipantDataWithRange() { - Map queryConditions = new HashMap<>(); - Map allParticipantDataForFiltering = new HashMap<>(); - Filter filter = new Filter(false, true, false, false, "DATE", "participantData", - new NameValue("PARTICIPANT_DEATH_DATE", "2040-10-29"), - new NameValue("PARTICIPANT_DEATH_DATE", "2040-10-31"), null, null); - List allParticipantData = participantDataDao - .getParticipantDataByInstanceId(Integer.parseInt(String.valueOf(ddpInstanceDto.getDdpInstanceId()))); - emptyFilterParticipantList.addParticipantDataIdsForFilters(filter, filter.getFilter1().getName(), allParticipantData, allParticipantDataForFiltering); - emptyFilterParticipantList.addParticipantDataConditionsToQuery(allParticipantDataForFiltering, queryConditions, 1); - Assert.assertEquals(" AND (profile.guid = " + participantId + " OR profile.guid = " + participantId1 + ")", - queryConditions.get(ElasticSearchUtil.ES)); - } - - @Test - public void testAddOneParticipantDataWithRangeFalse() { - Map queryConditions = new HashMap<>(); - Map allParticipantDataForFiltering = new HashMap<>(); - Filter filter = new Filter(false, true, false, false, "DATE", "participantData", - new NameValue("PARTICIPANT_DEATH_DATE", "2020-10-29"), - new NameValue("PARTICIPANT_DEATH_DATE", "2020-10-31"), null, null); - List allParticipantData = participantDataDao - .getParticipantDataByInstanceId(Integer.parseInt(String.valueOf(ddpInstanceDto.getDdpInstanceId()))); - emptyFilterParticipantList.addParticipantDataIdsForFilters(filter, filter.getFilter1().getName(), allParticipantData, allParticipantDataForFiltering); - emptyFilterParticipantList.addParticipantDataConditionsToQuery(allParticipantDataForFiltering, queryConditions, 1); - Assert.assertEquals("profile.guid = empty", queryConditions.get(ElasticSearchUtil.ES)); - } - - @Test - public void testAddSeveralParticipantDataFilter() { - Map queryConditions = new HashMap<>(); - Map queryConditions1 = new HashMap<>(); - Map allParticipantDataForFiltering = new HashMap<>(); - Filter filterA = new Filter(false, true, false, false, "DATE", "participantData", - new NameValue("PARTICIPANT_DEATH_DATE", "2040-10-30"), null, null, null); - Filter filterB = new Filter(false, true, false, false, "DATE", "participantData", - new NameValue("PARTICIPANT_DEATH_DATE", "2040-10-30"), null, null, null); - Filter filterC = new Filter(false, true, false, false, "DATE", "participantData", - new NameValue("PARTICIPANT_DEATH_DATE", "2040-10-30"), null, null, null); - List allParticipantData = participantDataDao - .getParticipantDataByInstanceId(Integer.parseInt(String.valueOf(ddpInstanceDto.getDdpInstanceId()))); - emptyFilterParticipantList.addParticipantDataIdsForFilters(filterA, filterA.getFilter1().getName(), allParticipantData, allParticipantDataForFiltering); - emptyFilterParticipantList.addParticipantDataIdsForFilters(filterB, filterB.getFilter1().getName(), allParticipantData, allParticipantDataForFiltering); - emptyFilterParticipantList.addParticipantDataIdsForFilters(filterC, filterC.getFilter1().getName(), allParticipantData, allParticipantDataForFiltering); - emptyFilterParticipantList.addParticipantDataConditionsToQuery(allParticipantDataForFiltering, queryConditions, 3); - emptyFilterParticipantList.addParticipantDataConditionsToQuery(allParticipantDataForFiltering, queryConditions1, 1); - Assert.assertEquals(" AND (profile.guid = RBMJW6ZIXVXBMXUX6M3Q" + " OR profile.guid = " + participantId1 + ")", - queryConditions.get(ElasticSearchUtil.ES)); - Assert.assertEquals(" AND ()", queryConditions1.get(ElasticSearchUtil.ES)); - } - - @AfterClass - public static void finish() { - if (participantDataId > 0) { - participantDataDao.delete(participantDataId); - } - if (participantDataId1 > 0) { - participantDataDao.delete(participantDataId1); - } - userDao.delete(userDto.getId()); - ddpInstanceDao.delete(ddpInstanceDto.getDdpInstanceId()); - } -} From 48ec3e2325cbb8558ab05e0a6a73248433902ab0 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 17 Jan 2022 12:24:18 +0400 Subject: [PATCH 319/405] DDP-7121 | change way of extracting operator from filter --- .../org/broadinstitute/dsm/model/Filter.java | 8 ++- .../dsm/model/elastic/filter/Operator.java | 67 +++++++++++++++---- .../BaseFilterParticipantList.java | 22 ++++-- .../model/elastic/filter/OperatorTest.java | 28 ++++++++ 4 files changed, 105 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/Filter.java b/src/main/java/org/broadinstitute/dsm/model/Filter.java index 236b6d8a0..d04454ccb 100644 --- a/src/main/java/org/broadinstitute/dsm/model/Filter.java +++ b/src/main/java/org/broadinstitute/dsm/model/Filter.java @@ -13,6 +13,7 @@ import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.db.structure.SqlDateConverter; +import org.broadinstitute.dsm.model.filter.participant.BaseFilterParticipantList; import org.broadinstitute.dsm.statics.DBConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -273,11 +274,12 @@ else if (filter.isNotEmpty()) { if (filter.getFilter1() != null && filter.getFilter1().getValue() != null && StringUtils.isNotBlank(String.valueOf( filter.getFilter1().getValue()))) { query = AND + jsonExtract + filter.getParentName() + DBConstants.ALIAS_DELIMITER + dbElement.getColumnName() + " , '$." + filter.getFilter2().getName() + "' ) "; - if (filter.isExactMatch()) { + if (BaseFilterParticipantList.isDateRange(filter)) { + System.out.println(); + } else if (filter.isExactMatch()) { query += EQUALS + "'#'"; query = query.replaceAll("#", String.valueOf(filter.getFilter1().getValue())); - } - else { + } else { query += " " + LIKE + " '%#%'"; query = query.replaceAll("#", String.valueOf(filter.getFilter1().getValue())); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java index 2fcbf6dd0..a77d719f8 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java @@ -1,30 +1,37 @@ package org.broadinstitute.dsm.model.elastic.filter; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.regex.Pattern; import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.Filter; public enum Operator { - LIKE(Filter.LIKE_TRIMMED), - EQUALS(Filter.EQUALS_TRIMMED), - GREATER_THAN_EQUALS(Filter.LARGER_EQUALS_TRIMMED), - LESS_THAN_EQUALS(Filter.SMALLER_EQUALS_TRIMMED), - IS_NOT_NULL(Filter.IS_NOT_NULL_TRIMMED), - DIAMOND_EQUALS(Filter.DIAMOND_EQUALS), - MULTIPLE_OPTIONS(Filter.OPEN_PARENTHESIS), - STR_DATE(Filter.DATE_FORMAT), - DATE_GREATER(Filter.DATE_GREATER), - DATE_LESS(Filter.DATE_LESS), - JSON_EXTRACT(Filter.JSON_EXTRACT), - DATE(Filter.DATE); + LIKE(Filter.LIKE_TRIMMED, "((?!').)*(LIKE)"), + EQUALS(Filter.EQUALS_TRIMMED, "((?!(<|>|')).)*(=)"), + GREATER_THAN_EQUALS(Filter.LARGER_EQUALS_TRIMMED, ""), + LESS_THAN_EQUALS(Filter.SMALLER_EQUALS_TRIMMED, ""), + IS_NOT_NULL(Filter.IS_NOT_NULL_TRIMMED, ""), + DIAMOND_EQUALS(Filter.DIAMOND_EQUALS, ""), + MULTIPLE_OPTIONS(Filter.OPEN_PARENTHESIS, ""), + STR_DATE(Filter.DATE_FORMAT, ""), + DATE_GREATER(Filter.DATE_GREATER, ""), + DATE_LESS(Filter.DATE_LESS, ""), + JSON_EXTRACT(Filter.JSON_EXTRACT, ""), + DATE(Filter.DATE, ""); public static final String UNKNOWN_OPERATOR = "Unknown operator"; private String value; + private Pattern regex; - Operator(String value) { + Operator(String value, String regex) { this.value = value; + this.regex = Pattern.compile(regex); } public static Operator getOperator(String value) { @@ -51,4 +58,38 @@ private static boolean isMultipleOptions(String[] splittedFilter) { return splittedFilter.length > 0 && Filter.OPEN_PARENTHESIS.equals(splittedFilter[0]) && Filter.CLOSE_PARENTHESIS.equals(splittedFilter[splittedFilter.length - 1]); } + + public static List collectOperatorsFromFilter(String filter) { + List operators = new ArrayList<>(); + for (Operator operator : Operator.values()) { + int startIndex = filter.indexOf(operator.value); + if (startIndex == -1) continue; + String extractedOperator = filter.substring(startIndex, startIndex + operator.value.length()); + operators.add(extractedOperator); + } + return operators; + } + + public static Operator extractOperator(String filter) { + String[] splittedFilter = filter.split(Filter.SPACE); + Arrays.stream(splittedFilter) + .filter(StringUtils::isNotBlank) + .filter(word -> Arrays.stream(Operator.values()).anyMatch(op -> op.value.equals(word))) + .distinct() + .reduce((prev, curr) -> String.join(Filter.SPACE, prev, curr)); + + Operator operator = null; + for (Operator op: Operator.values()) { + int startIndex = filter.indexOf(op.value); + if (startIndex <= 0) continue; + String extractedOperator = filter.substring(0, startIndex + op.value.length()); + boolean isOperatorMatch = op.regex.matcher(extractedOperator).matches(); + if (isOperatorMatch) { + operator = op; + break; + } + } + if (Objects.isNull(operator)) throw new NoSuchElementException(UNKNOWN_OPERATOR); + return operator; + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java b/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java index d5264dbe2..f29f3f0ab 100644 --- a/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java +++ b/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java @@ -1,5 +1,6 @@ package org.broadinstitute.dsm.model.filter.participant; +import java.time.LocalDate; import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -100,12 +101,17 @@ protected ParticipantWrapperResult filterParticipantList(Filter[] filters, Map queryConditions, Filter filter, String tmpName) { + DBElement dbElement = new DBElement(DBConstants.DDP_PARTICIPANT_DATA, DBConstants.DDP_PARTICIPANT_DATA_ALIAS, null, + DBConstants.ADDITIONAL_VALUES_JSON); + if (isDateRange(filter)) { + filter.setType(Filter.ADDITIONAL_VALUES); + Filter.getQueryStringForFiltering(filter, dbElement); + System.out.println(); + } filter.setFilter1(new NameValue(ESObjectConstants.ADDITIONAL_VALUES_JSON, filter.getFilter1().getValue())); filter.setFilter2(new NameValue(Util.underscoresToCamelCase(tmpName), null)); filter.setParentName(DBConstants.DDP_PARTICIPANT_DATA_ALIAS); filter.setType(Filter.ADDITIONAL_VALUES); - DBElement dbElement = new DBElement(DBConstants.DDP_PARTICIPANT_DATA, DBConstants.DDP_PARTICIPANT_DATA_ALIAS, null, - DBConstants.ADDITIONAL_VALUES_JSON); if (Objects.nonNull(filter.getSelectedOptions())) { for (String selectedOption : filter.getSelectedOptions()) { filter.getFilter1().setValue(selectedOption); @@ -117,8 +123,16 @@ private void addParticipantDataQueryToQueryConditions(Map queryC } } else { queryConditions.put(DBConstants.DDP_PARTICIPANT_DATA_ALIAS, Filter.getQueryStringForFiltering(filter, dbElement)); - queryConditions.merge(DBConstants.DDP_PARTICIPANT_DATA_ALIAS, Filter.getQueryStringForFiltering(filter, - dbElement), (prev, curr) -> String.join(Filter.SPACE, prev, curr)); + } + } + + public static boolean isDateRange(Filter filter) { + try { + LocalDate.parse(String.valueOf(filter.getFilter1().getValue())); + LocalDate.parse(String.valueOf(filter.getFilter2().getValue())); + return true; + } catch (Exception e) { + return false; } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java index 2506bb212..c715fb557 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java @@ -2,8 +2,12 @@ import static org.junit.Assert.*; +import java.util.Arrays; +import java.util.List; + import org.broadinstitute.dsm.model.Filter; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Test; public class OperatorTest { @@ -48,4 +52,28 @@ public void extractJsonExtractValue() { Operator likeOperator = Operator.extract(filter); assertEquals(Operator.LIKE, likeOperator); } + + @Test + @Ignore + public void collectOperatorsFromFilter() { + String filter = "m.mr_received >= STR_TO_DATE('1964-01-14','%Y-%m-%d')"; + List operators = Operator.collectOperatorsFromFilter(filter); + assertEquals(Arrays.asList(">=", "STR_TO_DATE"), operators); + } + + @Test + public void extractLikeOperator() { + String filter = "m.mr_received LIKE 'someLIKEthing LIKE '"; + Operator operator = Operator.extractOperator(filter); + assertEquals(Operator.LIKE, operator); + } + + @Test + public void extractEqualsOperator() { + String filter = "m.mr_received = 'someLIKEthing LIKE'"; + String filter2 = "m.mr_received <= 18"; + Operator operator = Operator.extractOperator(filter); +// Operator operator2 = Operator.extractOperator(filter2); + assertEquals(Operator.EQUALS, operator); + } } \ No newline at end of file From a322fed093d07b287ad9b89235855fbd1aed54fd Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 17 Jan 2022 14:59:48 +0400 Subject: [PATCH 320/405] DDP-7121 | start implementing new approach for extracting proper Operator --- .../dsm/model/elastic/filter/Operator.java | 58 +++++++++++++------ .../model/elastic/filter/OperatorTest.java | 43 ++++++++++++++ 2 files changed, 83 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java index a77d719f8..fc3c90733 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java @@ -1,10 +1,6 @@ package org.broadinstitute.dsm.model.elastic.filter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.NoSuchElementException; -import java.util.Objects; +import java.util.*; import java.util.regex.Pattern; import org.apache.commons.lang3.StringUtils; @@ -72,24 +68,50 @@ public static List collectOperatorsFromFilter(String filter) { public static Operator extractOperator(String filter) { String[] splittedFilter = filter.split(Filter.SPACE); - Arrays.stream(splittedFilter) + if (isMultipleOptions(splittedFilter)) + return MULTIPLE_OPTIONS; + Optional maybeOperator = Arrays.stream(splittedFilter) .filter(StringUtils::isNotBlank) + .map(word -> { + String strOperator = StringUtils.EMPTY; + for (Operator operator : Operator.values()) { + int startIndex = word.indexOf(operator.value); + if (startIndex == -1) continue; + strOperator = word.substring(startIndex, startIndex + operator.value.length()); + return strOperator; + } + return strOperator; + }) .filter(word -> Arrays.stream(Operator.values()).anyMatch(op -> op.value.equals(word))) .distinct() .reduce((prev, curr) -> String.join(Filter.SPACE, prev, curr)); - - Operator operator = null; - for (Operator op: Operator.values()) { - int startIndex = filter.indexOf(op.value); - if (startIndex <= 0) continue; - String extractedOperator = filter.substring(0, startIndex + op.value.length()); - boolean isOperatorMatch = op.regex.matcher(extractedOperator).matches(); - if (isOperatorMatch) { - operator = op; - break; + if (maybeOperator.isPresent()) { + String operator = maybeOperator.get(); + switch (operator) { + case "STR_TO_DATE =": + return Operator.STR_DATE; + case "STR_TO_DATE <=": + return Operator.DATE_LESS; + case "STR_TO_DATE >=": + return Operator.DATE_GREATER; + default: + return Operator.getOperator(operator); } + } else { + throw new NoSuchElementException(UNKNOWN_OPERATOR); } - if (Objects.isNull(operator)) throw new NoSuchElementException(UNKNOWN_OPERATOR); - return operator; +// Operator operator = null; +// for (Operator op: Operator.values()) { +// int startIndex = filter.indexOf(op.value); +// if (startIndex <= 0) continue; +// String extractedOperator = filter.substring(0, startIndex + op.value.length()); +// boolean isOperatorMatch = op.regex.matcher(extractedOperator).matches(); +// if (isOperatorMatch) { +// operator = op; +// break; +// } +// } +// if (Objects.isNull(operator)) throw new NoSuchElementException(UNKNOWN_OPERATOR); +// return operator; } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java index c715fb557..18f55b722 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java @@ -76,4 +76,47 @@ public void extractEqualsOperator() { // Operator operator2 = Operator.extractOperator(filter2); assertEquals(Operator.EQUALS, operator); } + + @Test + public void extractGreaterThanEqualsOperator() { + String filter = "m.mr_received >= 15"; + Operator operator = Operator.extractOperator(filter); + assertEquals(Operator.GREATER_THAN_EQUALS, operator); + } + + @Test + public void extractLessThanEqualsOperator() { + String filter = "m.mr_received <= 15"; + Operator operator = Operator.extractOperator(filter); + assertEquals(Operator.LESS_THAN_EQUALS, operator); + } + + @Test + @Ignore + public void extractIsNotNullOperator() { + String filter = "m.mr_received IS NOT NULL"; + Operator operator = Operator.extractOperator(filter); + assertEquals(Operator.IS_NOT_NULL, operator); + } + + @Test + public void extractDiamondEqualsOperatorOperator() { + String filter = "m.mr_received <=> 15"; + Operator operator = Operator.extractOperator(filter); + assertEquals(Operator.DIAMOND_EQUALS, operator); + } + + @Test + public void extractMultipleOptionsOperator() { + String filter = "(m.mr_received = 15 OR m.mr_received = 15)"; + Operator operator = Operator.extractOperator(filter); + assertEquals(Operator.MULTIPLE_OPTIONS, operator); + } + + @Test + public void extractStrToDateOperator() { + String filter = "STR_TO_DATE(m.fax_sent,'%Y-%m-%d') = STR_TO_DATE('2021-12-17','%Y-%m-%d')"; + Operator operator = Operator.extractOperator(filter); + assertEquals(Operator.STR_DATE, operator); + } } \ No newline at end of file From 95fd886ef7161f071cd8232e37839075e4ea4300 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 17 Jan 2022 16:23:11 +0400 Subject: [PATCH 321/405] DDP-7121 | change extract method operator for more proper extracting --- .../org/broadinstitute/dsm/model/Filter.java | 2 + .../elastic/filter/AndOrFilterSeparator.java | 6 +- .../dsm/model/elastic/filter/Operator.java | 126 ++++++++---------- .../model/elastic/filter/OperatorTest.java | 58 +++++--- .../splitter/DiamondEqualsSplitterTest.java | 5 +- .../splitter/JsonExtractSplitterTest.java | 2 - .../filter/splitter/LikeSplitterTest.java | 2 - .../splitter/MultipleOptionsSplitterTest.java | 5 +- .../filter/splitter/StrDateSplitterTest.java | 2 - 9 files changed, 105 insertions(+), 103 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/Filter.java b/src/main/java/org/broadinstitute/dsm/model/Filter.java index d04454ccb..65be9df80 100644 --- a/src/main/java/org/broadinstitute/dsm/model/Filter.java +++ b/src/main/java/org/broadinstitute/dsm/model/Filter.java @@ -61,6 +61,8 @@ public class Filter { public static final String DATE_FORMAT = "STR_TO_DATE"; public static final String DATE_GREATER = LARGER_EQUALS + DATE_FORMAT; public static final String DATE_LESS = SMALLER_EQUALS + DATE_FORMAT; + public static final char OPEN_PARENTHESIS_CHAR = '('; + public static final char CLOSE_PARENTHESIS_CHAR = ')'; public static String TEXT = "TEXT"; public static String OPTIONS = "OPTIONS"; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java index 735d3ddab..85628e507 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/AndOrFilterSeparator.java @@ -104,16 +104,14 @@ private String getAliasRegexByOperator(String operator) { } private boolean isOperatorWrappedInParenthesis(int startIndex) { - final char openParenthesis = '('; - final char closeParenthesis = ')'; boolean exists = false; for (int i = startIndex; i > 2; i--) { char c = filter.charAt(i); - if (c == openParenthesis) { + if (c == Filter.OPEN_PARENTHESIS_CHAR) { exists = true; break; } - if (Filter.AND_TRIMMED.equals(filter.substring(i - 3, i)) || c == closeParenthesis) break; + if (Filter.AND_TRIMMED.equals(filter.substring(i - 3, i)) || c == Filter.CLOSE_PARENTHESIS_CHAR) break; } return exists; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java index fc3c90733..266b7ffed 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java @@ -2,37 +2,38 @@ import java.util.*; import java.util.regex.Pattern; +import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.Filter; public enum Operator { - LIKE(Filter.LIKE_TRIMMED, "((?!').)*(LIKE)"), - EQUALS(Filter.EQUALS_TRIMMED, "((?!(<|>|')).)*(=)"), - GREATER_THAN_EQUALS(Filter.LARGER_EQUALS_TRIMMED, ""), - LESS_THAN_EQUALS(Filter.SMALLER_EQUALS_TRIMMED, ""), - IS_NOT_NULL(Filter.IS_NOT_NULL_TRIMMED, ""), - DIAMOND_EQUALS(Filter.DIAMOND_EQUALS, ""), - MULTIPLE_OPTIONS(Filter.OPEN_PARENTHESIS, ""), - STR_DATE(Filter.DATE_FORMAT, ""), - DATE_GREATER(Filter.DATE_GREATER, ""), - DATE_LESS(Filter.DATE_LESS, ""), - JSON_EXTRACT(Filter.JSON_EXTRACT, ""), - DATE(Filter.DATE, ""); + LIKE(Filter.LIKE_TRIMMED), + EQUALS(Filter.EQUALS_TRIMMED), + GREATER_THAN_EQUALS(Filter.LARGER_EQUALS_TRIMMED), + LESS_THAN_EQUALS(Filter.SMALLER_EQUALS_TRIMMED), + IS_NOT_NULL(Filter.IS_NOT_NULL_TRIMMED), + DIAMOND_EQUALS(Filter.DIAMOND_EQUALS), + MULTIPLE_OPTIONS(Operator.MULTIPLE_OPTIONS_INDICATOR), + STR_DATE(Filter.DATE_FORMAT), + DATE_GREATER(Filter.DATE_GREATER), + DATE_LESS(Filter.DATE_LESS), + JSON_EXTRACT(Filter.JSON_EXTRACT), + DATE(Filter.DATE); + + public static final String MULTIPLE_OPTIONS_INDICATOR = "()"; public static final String UNKNOWN_OPERATOR = "Unknown operator"; private String value; - private Pattern regex; - Operator(String value, String regex) { + Operator(String value) { this.value = value; - this.regex = Pattern.compile(regex); } public static Operator getOperator(String value) { for (Operator op: Operator.values()) { - if (op.value.equals(value)) return op; + if (op.value.trim().equals(value)) return op; } throw new IllegalArgumentException(UNKNOWN_OPERATOR); } @@ -41,47 +42,11 @@ public static Operator extract(String filter) { String[] splittedFilter = filter.split(Filter.SPACE); if (isMultipleOptions(splittedFilter)) return MULTIPLE_OPTIONS; - String operator = Arrays.stream(splittedFilter) - .filter(StringUtils::isNotBlank) - .filter(str -> !Filter.OPEN_PARENTHESIS.equals(str) && !Filter.CLOSE_PARENTHESIS.equals(str)) - .filter(str -> Arrays.stream(Operator.values()).anyMatch(op -> op.value.equals(str))) - .findFirst() - .orElse(StringUtils.EMPTY); - return getOperator(operator); - } - - private static boolean isMultipleOptions(String[] splittedFilter) { - return splittedFilter.length > 0 && Filter.OPEN_PARENTHESIS.equals(splittedFilter[0]) && - Filter.CLOSE_PARENTHESIS.equals(splittedFilter[splittedFilter.length - 1]); - } - - public static List collectOperatorsFromFilter(String filter) { - List operators = new ArrayList<>(); - for (Operator operator : Operator.values()) { - int startIndex = filter.indexOf(operator.value); - if (startIndex == -1) continue; - String extractedOperator = filter.substring(startIndex, startIndex + operator.value.length()); - operators.add(extractedOperator); - } - return operators; - } - - public static Operator extractOperator(String filter) { - String[] splittedFilter = filter.split(Filter.SPACE); - if (isMultipleOptions(splittedFilter)) - return MULTIPLE_OPTIONS; + else if (isNotNull(splittedFilter)) + return IS_NOT_NULL; Optional maybeOperator = Arrays.stream(splittedFilter) .filter(StringUtils::isNotBlank) - .map(word -> { - String strOperator = StringUtils.EMPTY; - for (Operator operator : Operator.values()) { - int startIndex = word.indexOf(operator.value); - if (startIndex == -1) continue; - strOperator = word.substring(startIndex, startIndex + operator.value.length()); - return strOperator; - } - return strOperator; - }) + .map(Operator::handleSpecialCaseOperators) .filter(word -> Arrays.stream(Operator.values()).anyMatch(op -> op.value.equals(word))) .distinct() .reduce((prev, curr) -> String.join(Filter.SPACE, prev, curr)); @@ -94,24 +59,49 @@ public static Operator extractOperator(String filter) { return Operator.DATE_LESS; case "STR_TO_DATE >=": return Operator.DATE_GREATER; + case "DATE =": + return Operator.DATE; default: return Operator.getOperator(operator); } } else { throw new NoSuchElementException(UNKNOWN_OPERATOR); } -// Operator operator = null; -// for (Operator op: Operator.values()) { -// int startIndex = filter.indexOf(op.value); -// if (startIndex <= 0) continue; -// String extractedOperator = filter.substring(0, startIndex + op.value.length()); -// boolean isOperatorMatch = op.regex.matcher(extractedOperator).matches(); -// if (isOperatorMatch) { -// operator = op; -// break; -// } -// } -// if (Objects.isNull(operator)) throw new NoSuchElementException(UNKNOWN_OPERATOR); -// return operator; + } + + private static String handleSpecialCaseOperators(String word) { + String strOperator = StringUtils.EMPTY; + for (Operator operator : Operator.values()) { + int startIndex = word.indexOf(operator.value); + if (startIndex == -1) continue; + if (word.contains(Filter.OPEN_PARENTHESIS)) strOperator = word.substring(startIndex, startIndex + operator.value.length()); + else strOperator = word; + return strOperator; + } + return strOperator; + } + + private static boolean isMultipleOptions(String[] splittedFilter) { + splittedFilter = cleanFromEmptySpaces(splittedFilter); + if (splittedFilter.length == 0) return false; + String firstElement = splittedFilter[0]; + String lastElement = splittedFilter[splittedFilter.length - 1]; + return (Filter.OPEN_PARENTHESIS.equals(firstElement) && Filter.CLOSE_PARENTHESIS.equals(lastElement)) + || (firstElement.charAt(0) == Filter.OPEN_PARENTHESIS_CHAR && lastElement.charAt(lastElement.length()-1) == + Filter.CLOSE_PARENTHESIS_CHAR); + } + + private static String[] cleanFromEmptySpaces(String[] splittedFilter) { + return Arrays.stream(splittedFilter) + .filter(StringUtils::isNotBlank) + .collect(Collectors.toList()) + .toArray(new String[] {}); + } + + private static boolean isNotNull(String[] splittedFilter) { + splittedFilter = cleanFromEmptySpaces(splittedFilter); + final int BOTTOM_SIZE_OF_IS_NOT_NULL = 4; + if (splittedFilter.length < BOTTOM_SIZE_OF_IS_NOT_NULL) return false; + return Filter.IS.equals(splittedFilter[1]) && Filter.NOT.equals(splittedFilter[2]) && Filter.NULL.equals(splittedFilter[3]); } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java index 18f55b722..91312f5df 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java @@ -53,70 +53,90 @@ public void extractJsonExtractValue() { assertEquals(Operator.LIKE, likeOperator); } - @Test - @Ignore - public void collectOperatorsFromFilter() { - String filter = "m.mr_received >= STR_TO_DATE('1964-01-14','%Y-%m-%d')"; - List operators = Operator.collectOperatorsFromFilter(filter); - assertEquals(Arrays.asList(">=", "STR_TO_DATE"), operators); - } - @Test public void extractLikeOperator() { String filter = "m.mr_received LIKE 'someLIKEthing LIKE '"; - Operator operator = Operator.extractOperator(filter); + Operator operator = Operator.extract(filter); assertEquals(Operator.LIKE, operator); } @Test public void extractEqualsOperator() { String filter = "m.mr_received = 'someLIKEthing LIKE'"; - String filter2 = "m.mr_received <= 18"; - Operator operator = Operator.extractOperator(filter); -// Operator operator2 = Operator.extractOperator(filter2); + Operator operator = Operator.extract(filter); assertEquals(Operator.EQUALS, operator); } @Test public void extractGreaterThanEqualsOperator() { String filter = "m.mr_received >= 15"; - Operator operator = Operator.extractOperator(filter); + Operator operator = Operator.extract(filter); assertEquals(Operator.GREATER_THAN_EQUALS, operator); } @Test public void extractLessThanEqualsOperator() { String filter = "m.mr_received <= 15"; - Operator operator = Operator.extractOperator(filter); + Operator operator = Operator.extract(filter); assertEquals(Operator.LESS_THAN_EQUALS, operator); } @Test - @Ignore public void extractIsNotNullOperator() { String filter = "m.mr_received IS NOT NULL"; - Operator operator = Operator.extractOperator(filter); + Operator operator = Operator.extract(filter); assertEquals(Operator.IS_NOT_NULL, operator); } @Test public void extractDiamondEqualsOperatorOperator() { String filter = "m.mr_received <=> 15"; - Operator operator = Operator.extractOperator(filter); + Operator operator = Operator.extract(filter); assertEquals(Operator.DIAMOND_EQUALS, operator); } @Test public void extractMultipleOptionsOperator() { String filter = "(m.mr_received = 15 OR m.mr_received = 15)"; - Operator operator = Operator.extractOperator(filter); + String filter2 = " ( m.mr_received = 15 OR m.mr_received = 15 ) "; + Operator operator = Operator.extract(filter); + Operator operator2 = Operator.extract(filter2); assertEquals(Operator.MULTIPLE_OPTIONS, operator); + assertEquals(Operator.MULTIPLE_OPTIONS, operator2); } @Test public void extractStrToDateOperator() { String filter = "STR_TO_DATE(m.fax_sent,'%Y-%m-%d') = STR_TO_DATE('2021-12-17','%Y-%m-%d')"; - Operator operator = Operator.extractOperator(filter); + Operator operator = Operator.extract(filter); assertEquals(Operator.STR_DATE, operator); } + + @Test + public void extractGreaterStrToDateOperator() { + String filter = "m.mr_received >= STR_TO_DATE('1964-01-14','%Y-%m-%d')"; + Operator operator = Operator.extract(filter); + assertEquals(Operator.DATE_GREATER, operator); + } + + @Test + public void extractLessStrToDateOperator() { + String filter = "m.mr_received <= STR_TO_DATE('1964-01-14','%Y-%m-%d')"; + Operator operator = Operator.extract(filter); + assertEquals(Operator.DATE_LESS, operator); + } + + @Test + public void extractJsonExctractOperator() { + String filter = "JSON_EXTRACT ( m.additiona`l_values_json , '$.seeingIfBugExists' )"; + Operator operator = Operator.extract(filter); + assertEquals(Operator.JSON_EXTRACT, operator); + } + + @Test + public void extractDateOperator() { + String filter = "DATE(FROM_UNIXTIME(k.scan_date/1000)) = DATE(FROM_UNIXTIME(1640563200))"; + Operator operator = Operator.extract(filter); + assertEquals(Operator.DATE, operator); + } } \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DiamondEqualsSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DiamondEqualsSplitterTest.java index 4a3de4b0f..5878d6e24 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DiamondEqualsSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DiamondEqualsSplitterTest.java @@ -1,9 +1,8 @@ package org.broadinstitute.dsm.model.elastic.filter.splitter; -import org.broadinstitute.dsm.model.elastic.filter.splitter.DiamondEqualsSplitter; -import org.junit.Test; +import static org.junit.Assert.assertEquals; -import static org.junit.Assert.*; +import org.junit.Test; public class DiamondEqualsSplitterTest { diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonExtractSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonExtractSplitterTest.java index 4b7170ae0..ce4f0af2c 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonExtractSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonExtractSplitterTest.java @@ -2,8 +2,6 @@ import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.elastic.filter.Operator; -import org.broadinstitute.dsm.model.elastic.filter.splitter.BaseSplitter; -import org.broadinstitute.dsm.model.elastic.filter.splitter.SplitterFactory; import org.junit.Assert; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/LikeSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/LikeSplitterTest.java index 5fb5fd621..8b66894f1 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/LikeSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/LikeSplitterTest.java @@ -4,8 +4,6 @@ import org.junit.Assert; import org.junit.Test; -import static org.junit.Assert.*; - public class LikeSplitterTest { @Test diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/MultipleOptionsSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/MultipleOptionsSplitterTest.java index 515d81143..670c20da2 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/MultipleOptionsSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/MultipleOptionsSplitterTest.java @@ -1,10 +1,9 @@ package org.broadinstitute.dsm.model.elastic.filter.splitter; -import static org.junit.Assert.*; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; import org.broadinstitute.dsm.model.elastic.filter.Operator; -import org.broadinstitute.dsm.model.elastic.filter.splitter.BaseSplitter; -import org.broadinstitute.dsm.model.elastic.filter.splitter.SplitterFactory; import org.junit.Test; public class MultipleOptionsSplitterTest { diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/StrDateSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/StrDateSplitterTest.java index 2965d9d5d..15c0f9881 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/StrDateSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/StrDateSplitterTest.java @@ -1,8 +1,6 @@ package org.broadinstitute.dsm.model.elastic.filter.splitter; import org.broadinstitute.dsm.model.elastic.filter.Operator; -import org.broadinstitute.dsm.model.elastic.filter.splitter.BaseSplitter; -import org.broadinstitute.dsm.model.elastic.filter.splitter.SplitterFactory; import org.junit.Assert; import org.junit.Test; From 21b983ff14cd128e82f77684de73777fc65d746a Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 17 Jan 2022 19:04:20 +0400 Subject: [PATCH 322/405] DDP-7121 | fix bug for dynamic fields date range --- .../org/broadinstitute/dsm/model/Filter.java | 10 ++++++++-- .../dsm/model/elastic/filter/Operator.java | 4 ++++ .../filter/query/DsmAbstractQueryBuilder.java | 2 +- .../filter/query/QueryBuilderFactory.java | 18 ++++++++++++++++-- .../filter/splitter/JsonExtractSplitter.java | 5 ++++- .../participant/BaseFilterParticipantList.java | 16 ++++++++++------ .../dsm/model/elastic/filter/OperatorTest.java | 2 +- .../filter/query/QueryBuilderFactoryTest.java | 8 +++----- 8 files changed, 47 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/Filter.java b/src/main/java/org/broadinstitute/dsm/model/Filter.java index 65be9df80..b229bb23c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/Filter.java +++ b/src/main/java/org/broadinstitute/dsm/model/Filter.java @@ -277,7 +277,13 @@ else if (filter.isNotEmpty()) { filter.getFilter1().getValue()))) { query = AND + jsonExtract + filter.getParentName() + DBConstants.ALIAS_DELIMITER + dbElement.getColumnName() + " , '$." + filter.getFilter2().getName() + "' ) "; if (BaseFilterParticipantList.isDateRange(filter)) { - System.out.println(); + String moreThan = generateDateComparisonSql(filter, dbElement, LARGER_EQUALS, filter.getFilter1().getValue(), false); + String lessThan = generateDateComparisonSql(filter, dbElement, SMALLER_EQUALS, filter.getFilter2().getValue(), true); + int moreThanIndex = moreThan.indexOf(Filter.LARGER_EQUALS_TRIMMED); + int lessThanIndex = lessThan.indexOf(Filter.SMALLER_EQUALS_TRIMMED); + moreThan = moreThan.substring(moreThanIndex); + lessThan = lessThan.substring(lessThanIndex); + query += moreThan + query + lessThan; } else if (filter.isExactMatch()) { query += EQUALS + "'#'"; query = query.replaceAll("#", String.valueOf(filter.getFilter1().getValue())); @@ -286,7 +292,7 @@ else if (filter.isNotEmpty()) { query = query.replaceAll("#", String.valueOf(filter.getFilter1().getValue())); } } - if (Objects.nonNull(filter.getSelectedOptions())) { + if (Objects.nonNull(filter.getSelectedOptions()) || BaseFilterParticipantList.isDateRange(filter)) { finalQuery = query; } else { finalQuery = notNullQuery + query; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java index 266b7ffed..6d396cbf1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java @@ -61,6 +61,10 @@ else if (isNotNull(splittedFilter)) return Operator.DATE_GREATER; case "DATE =": return Operator.DATE; + case "JSON_EXTRACT =": + case "JSON_EXTRACT >=": + case "JSON_EXTRACT <=": + return JSON_EXTRACT; default: return Operator.getOperator(operator); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/DsmAbstractQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/DsmAbstractQueryBuilder.java index 684d96d0e..2283076c7 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/DsmAbstractQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/DsmAbstractQueryBuilder.java @@ -73,7 +73,7 @@ protected void buildUpQuery(List filterValues, FilterStrategy filterStra splitter = SplitterFactory.createSplitter(operator, filterValue); splitter.setFilter(filterValue); QueryPayload queryPayload = new QueryPayload(buildPath(), splitter.getInnerProperty(), parser.parse(splitter.getValue())); - queryBuilder = QueryBuilderFactory.buildQueryBuilder(operator, queryPayload); + queryBuilder = QueryBuilderFactory.buildQueryBuilder(operator, queryPayload, splitter); buildEachQuery(filterStrategy); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryBuilderFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryBuilderFactory.java index 32af7434c..53324b6e3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryBuilderFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryBuilderFactory.java @@ -2,6 +2,10 @@ import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.elastic.filter.Operator; +import org.broadinstitute.dsm.model.elastic.filter.splitter.BaseSplitter; +import org.broadinstitute.dsm.model.elastic.filter.splitter.GreaterThanEqualsSplitter; +import org.broadinstitute.dsm.model.elastic.filter.splitter.JsonExtractSplitter; +import org.broadinstitute.dsm.model.elastic.filter.splitter.LessThanEqualsSplitter; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.ExistsQueryBuilder; import org.elasticsearch.index.query.MatchQueryBuilder; @@ -9,7 +13,8 @@ import org.elasticsearch.index.query.RangeQueryBuilder; public class QueryBuilderFactory { - public static QueryBuilder buildQueryBuilder(Operator operator, QueryPayload payload) { + public static QueryBuilder buildQueryBuilder(Operator operator, QueryPayload payload, + BaseSplitter splitter) { QueryBuilder qb; switch (operator) { case LIKE: @@ -53,7 +58,16 @@ public static QueryBuilder buildQueryBuilder(Operator operator, QueryPayload pay case JSON_EXTRACT: Object[] dynamicFieldValues = payload.getValues(); if (!StringUtils.EMPTY.equals(dynamicFieldValues[0])) { - qb = new MatchQueryBuilder(payload.getFieldName(), dynamicFieldValues[0]); + JsonExtractSplitter jsonExtractSplitter = (JsonExtractSplitter) splitter; + if (jsonExtractSplitter.getDecoratedSplitter() instanceof GreaterThanEqualsSplitter) { + qb = new RangeQueryBuilder(payload.getFieldName()); + ((RangeQueryBuilder)qb).gte(dynamicFieldValues[0]); + } else if (jsonExtractSplitter.getDecoratedSplitter() instanceof LessThanEqualsSplitter) { + qb = new RangeQueryBuilder(payload.getFieldName()); + ((RangeQueryBuilder)qb).lte(dynamicFieldValues[0]); + } else { + qb = new MatchQueryBuilder(payload.getFieldName(), dynamicFieldValues[0]); + } } else { qb = new ExistsQueryBuilder(payload.getFieldName()); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonExtractSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonExtractSplitter.java index a1614d337..576cacbd7 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonExtractSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonExtractSplitter.java @@ -11,7 +11,6 @@ public class JsonExtractSplitter extends BaseSplitter { private BaseSplitter decoratedSplitter; - public JsonExtractSplitter(BaseSplitter splitter) { this.decoratedSplitter = splitter; } @@ -20,6 +19,10 @@ public JsonExtractSplitter() { decoratedSplitter = new EqualsSplitter(); } + public BaseSplitter getDecoratedSplitter() { + return decoratedSplitter; + } + @Override public String[] split() { decoratedSplitter.filter = filter; diff --git a/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java b/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java index f29f3f0ab..3ebfd2e16 100644 --- a/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java +++ b/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java @@ -104,15 +104,19 @@ private void addParticipantDataQueryToQueryConditions(Map queryC DBElement dbElement = new DBElement(DBConstants.DDP_PARTICIPANT_DATA, DBConstants.DDP_PARTICIPANT_DATA_ALIAS, null, DBConstants.ADDITIONAL_VALUES_JSON); if (isDateRange(filter)) { + filter.getFilter1().setName(ESObjectConstants.ADDITIONAL_VALUES_JSON); + filter.getFilter2().setName(Util.underscoresToCamelCase(tmpName)); + filter.setNotEmpty(false); + filter.setParentName(DBConstants.DDP_PARTICIPANT_DATA_ALIAS); filter.setType(Filter.ADDITIONAL_VALUES); Filter.getQueryStringForFiltering(filter, dbElement); - System.out.println(); + } else { + filter.setFilter1(new NameValue(ESObjectConstants.ADDITIONAL_VALUES_JSON, filter.getFilter1().getValue())); + filter.setFilter2(new NameValue(Util.underscoresToCamelCase(tmpName), null)); + filter.setParentName(DBConstants.DDP_PARTICIPANT_DATA_ALIAS); + filter.setType(Filter.ADDITIONAL_VALUES); } - filter.setFilter1(new NameValue(ESObjectConstants.ADDITIONAL_VALUES_JSON, filter.getFilter1().getValue())); - filter.setFilter2(new NameValue(Util.underscoresToCamelCase(tmpName), null)); - filter.setParentName(DBConstants.DDP_PARTICIPANT_DATA_ALIAS); - filter.setType(Filter.ADDITIONAL_VALUES); - if (Objects.nonNull(filter.getSelectedOptions())) { + if (Objects.nonNull(filter.getSelectedOptions()) && filter.getSelectedOptions().length > 0) { for (String selectedOption : filter.getSelectedOptions()) { filter.getFilter1().setValue(selectedOption); filter.getFilter2().setName(Util.underscoresToCamelCase(tmpName)); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java index 91312f5df..62121d27d 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java @@ -128,7 +128,7 @@ public void extractLessStrToDateOperator() { @Test public void extractJsonExctractOperator() { - String filter = "JSON_EXTRACT ( m.additiona`l_values_json , '$.seeingIfBugExists' )"; + String filter = "JSON_EXTRACT ( m.additiona`l_values_json , '$.seeingIfBugExists' ) = 'true'"; Operator operator = Operator.extract(filter); assertEquals(Operator.JSON_EXTRACT, operator); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryBuilderFactoryTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryBuilderFactoryTest.java index 6029c1eb7..0cc0641c3 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryBuilderFactoryTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryBuilderFactoryTest.java @@ -3,8 +3,6 @@ import static org.junit.Assert.*; import org.broadinstitute.dsm.model.elastic.filter.Operator; -import org.broadinstitute.dsm.model.elastic.filter.query.QueryBuilderFactory; -import org.broadinstitute.dsm.model.elastic.filter.query.QueryPayload; import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.RangeQueryBuilder; @@ -17,13 +15,13 @@ public class QueryBuilderFactoryTest { public void buildQueryBuilder() { QueryPayload payload = new QueryPayload("dsm.medicalRecord", "medicalRecordId", new Integer[] {10}); Operator operator = Operator.EQUALS; - QueryBuilder queryBuilder = QueryBuilderFactory.buildQueryBuilder(operator, payload); + QueryBuilder queryBuilder = QueryBuilderFactory.buildQueryBuilder(operator, payload, null); assertTrue(queryBuilder instanceof MatchQueryBuilder); operator = Operator.LIKE; - queryBuilder = QueryBuilderFactory.buildQueryBuilder(operator, payload); + queryBuilder = QueryBuilderFactory.buildQueryBuilder(operator, payload, null); assertTrue(queryBuilder instanceof MatchQueryBuilder); operator = Operator.GREATER_THAN_EQUALS; - queryBuilder = QueryBuilderFactory.buildQueryBuilder(operator, payload); + queryBuilder = QueryBuilderFactory.buildQueryBuilder(operator, payload, null); assertTrue(queryBuilder instanceof RangeQueryBuilder); } From 6e00e7fb4b72fe0ab00710649a7daed2fc78c840 Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 18 Jan 2022 14:49:36 +0400 Subject: [PATCH 323/405] DDP-7121 | compute proper tableAlias for searching, remove setting new parentname for filter --- src/main/java/org/broadinstitute/dsm/db/ViewFilter.java | 1 - .../filter/participant/BaseFilterParticipantList.java | 8 ++++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/ViewFilter.java b/src/main/java/org/broadinstitute/dsm/db/ViewFilter.java index 0745d9835..2ab89f5a7 100644 --- a/src/main/java/org/broadinstitute/dsm/db/ViewFilter.java +++ b/src/main/java/org/broadinstitute/dsm/db/ViewFilter.java @@ -271,7 +271,6 @@ public static void addQueryCondition(@NonNull Map queryCondition if (queryConditions.containsKey("ES")) { queryCondition = queryConditions.get("ES"); } - filter.setParentName(filter.getParticipantColumn().getTableAlias()); queryConditions.put("ES", queryCondition.concat(Filter.getQueryStringForFiltering(filter, null))); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java b/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java index 3ebfd2e16..c59374faa 100644 --- a/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java +++ b/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java @@ -58,10 +58,14 @@ protected ParticipantWrapperResult filterParticipantList(Filter[] filters, Map Date: Tue, 18 Jan 2022 16:38:34 +0400 Subject: [PATCH 324/405] DDP-7121 | fix filtering by date bug --- src/main/java/org/broadinstitute/dsm/db/ViewFilter.java | 1 + .../model/filter/participant/BaseFilterParticipantList.java | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/ViewFilter.java b/src/main/java/org/broadinstitute/dsm/db/ViewFilter.java index 2ab89f5a7..2b91a2680 100644 --- a/src/main/java/org/broadinstitute/dsm/db/ViewFilter.java +++ b/src/main/java/org/broadinstitute/dsm/db/ViewFilter.java @@ -533,6 +533,7 @@ else if (word.equals(Filter.JSON_CONTAINS)) {// type is JSONARRAY if (word.equals(Filter.EQUALS_TRIMMED)) { // exact match selected in the frontend exact = true; range = false; + f1 = true; state = 3; break; } diff --git a/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java b/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java index c59374faa..489b87c51 100644 --- a/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java +++ b/src/main/java/org/broadinstitute/dsm/model/filter/participant/BaseFilterParticipantList.java @@ -59,10 +59,11 @@ protected ParticipantWrapperResult filterParticipantList(Filter[] filters, Map Date: Tue, 18 Jan 2022 17:09:53 +0400 Subject: [PATCH 325/405] DDP-7121 | fix profile related bugs --- .../dsm/util/ElasticSearchUtil.java | 126 ++++++++++-------- 1 file changed, 70 insertions(+), 56 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/util/ElasticSearchUtil.java b/src/main/java/org/broadinstitute/dsm/util/ElasticSearchUtil.java index ed1e699a6..fed9fc073 100644 --- a/src/main/java/org/broadinstitute/dsm/util/ElasticSearchUtil.java +++ b/src/main/java/org/broadinstitute/dsm/util/ElasticSearchUtil.java @@ -873,62 +873,62 @@ public static AbstractQueryBuilder> createESQu if (nameValue[0].startsWith(PROFILE) || nameValue[0].startsWith(ADDRESS)) { try { long date = SystemUtil.getLongFromString(userEntered); - QueryBuilder tmpBuilder = findQueryBuilderForFieldName(finalQuery, nameValue[0]); + QueryBuilder tmpBuilder = findQueryBuilderForFieldName(finalQuery, nameValue[0].trim()); if (tmpBuilder != null) { ((RangeQueryBuilder) tmpBuilder).gte(date); } else { - finalQuery.must(QueryBuilders.rangeQuery(nameValue[0]).gte(date)); + finalQuery.must(QueryBuilders.rangeQuery(nameValue[0].trim()).gte(date)); } } catch (ParseException e) { - finalQuery.must(QueryBuilders.matchQuery(nameValue[0], userEntered)); + finalQuery.must(QueryBuilders.matchQuery(nameValue[0].trim(), userEntered)); } } else if (nameValue[0].startsWith(DSM)) { - QueryBuilder tmpBuilder = findQueryBuilderForFieldName(finalQuery, nameValue[0]); + QueryBuilder tmpBuilder = findQueryBuilderForFieldName(finalQuery, nameValue[0].trim()); if (tmpBuilder != null) { ((RangeQueryBuilder) tmpBuilder).gte(userEntered); } else { - finalQuery.must(QueryBuilders.rangeQuery(nameValue[0]).gte(userEntered)); + finalQuery.must(QueryBuilders.rangeQuery(nameValue[0].trim()).gte(userEntered)); } } else if (nameValue[0].startsWith(DATA)) { String[] dataParam = nameValue[0].split("\\."); - QueryBuilder tmpBuilder = findQueryBuilderForFieldName(finalQuery, dataParam[1]); + QueryBuilder tmpBuilder = findQueryBuilderForFieldName(finalQuery, dataParam[1].trim()); try { long date = SystemUtil.getLongFromString(userEntered); if (tmpBuilder != null) { ((RangeQueryBuilder) tmpBuilder).gte(date); } else { - finalQuery.must(QueryBuilders.rangeQuery(dataParam[1]).gte(date)); + finalQuery.must(QueryBuilders.rangeQuery(dataParam[1].trim()).gte(date)); } } catch (ParseException e) { logger.error("range was not date. user entered: " + userEntered); } } else if (nameValue[0].startsWith(INVITATIONS)) { String[] invitationParam = nameValue[0].split("\\."); - QueryBuilder tmpBuilder = findQueryBuilderForFieldName(finalQuery, invitationParam[1]); + QueryBuilder tmpBuilder = findQueryBuilderForFieldName(finalQuery, invitationParam[1].trim()); try { long date = SystemUtil.getLongFromString(userEntered); if (tmpBuilder != null) { ((RangeQueryBuilder) tmpBuilder).gte(date); } else { - finalQuery.must(QueryBuilders.rangeQuery(INVITATIONS + DBConstants.ALIAS_DELIMITER + invitationParam[1]).gte(date)); + finalQuery.must(QueryBuilders.rangeQuery(INVITATIONS + DBConstants.ALIAS_DELIMITER + invitationParam[1].trim()).gte(date)); } } catch (ParseException e) { logger.error("range was not date. user entered: " + userEntered); } } else { String[] surveyParam = nameValue[0].split("\\."); - if (CREATED_AT.equals(surveyParam[1]) || COMPLETED_AT.equals(surveyParam[1]) || LAST_UPDATED.equals(surveyParam[1])) { - QueryBuilder tmpBuilder = findQueryBuilderForFieldName(finalQuery, surveyParam[1]); + if (CREATED_AT.equals(surveyParam[1].trim()) || COMPLETED_AT.equals(surveyParam[1].trim()) || LAST_UPDATED.equals(surveyParam[1].trim())) { + QueryBuilder tmpBuilder = findQueryBuilderForFieldName(finalQuery, surveyParam[1].trim()); try { long date = SystemUtil.getLongFromString(userEntered); if (tmpBuilder != null) { ((RangeQueryBuilder) tmpBuilder).gte(date); } else { tmpBuilder = new BoolQueryBuilder(); - ((BoolQueryBuilder) tmpBuilder).must(QueryBuilders.rangeQuery(ACTIVITIES + DBConstants.ALIAS_DELIMITER + surveyParam[1]).gte(date)); + ((BoolQueryBuilder) tmpBuilder).must(QueryBuilders.rangeQuery(ACTIVITIES + DBConstants.ALIAS_DELIMITER + surveyParam[1].trim()).gte(date)); BoolQueryBuilder activityAnswer = new BoolQueryBuilder(); activityAnswer.must(tmpBuilder); - activityAnswer.must(QueryBuilders.matchQuery(ACTIVITIES + DBConstants.ALIAS_DELIMITER + ACTIVITY_CODE, surveyParam[0]).operator(Operator.AND)); + activityAnswer.must(QueryBuilders.matchQuery(ACTIVITIES + DBConstants.ALIAS_DELIMITER + ACTIVITY_CODE, surveyParam[0].trim()).operator(Operator.AND)); NestedQueryBuilder query = QueryBuilders.nestedQuery(ACTIVITIES, activityAnswer, ScoreMode.Avg); finalQuery.must(query); } @@ -937,7 +937,13 @@ public static AbstractQueryBuilder> createESQu } } else if (StringUtils.isNumeric(userEntered)) { // separately process expressions with numbers NestedQueryBuilder query = addRangeLimitForNumber( - Filter.LARGER_EQUALS, surveyParam[1], userEntered, finalQuery, queryPartsMap); + Filter.LARGER_EQUALS, surveyParam[1].trim(), userEntered, finalQuery, queryPartsMap, ACTIVITIES_QUESTIONS_ANSWER_ANSWER); + if (query != null) { + parentNestedOfRangeBuilderOfNumbers = query; + } + } else if (userEntered.trim().matches("\\d{4}-\\d{1,2}-\\d{1,2}")) { // separately process expressions with numbers + NestedQueryBuilder query = addRangeLimitForNumber( + Filter.LARGER_EQUALS, surveyParam[1].trim(), userEntered, finalQuery, queryPartsMap, ACTIVITIES_QUESTIONS_ANSWER_DATE); if (query != null) { parentNestedOfRangeBuilderOfNumbers = query; } @@ -954,59 +960,59 @@ public static AbstractQueryBuilder> createESQu if (nameValue[0].startsWith(PROFILE) || nameValue[0].startsWith(ADDRESS)) { String endDate = userEntered + END_OF_DAY; long date = SystemUtil.getLongFromDetailDateString(endDate); - QueryBuilder tmpBuilder = findQueryBuilderForFieldName(finalQuery, nameValue[0]); + QueryBuilder tmpBuilder = findQueryBuilderForFieldName(finalQuery, nameValue[0].trim()); if (tmpBuilder != null) { ((RangeQueryBuilder) tmpBuilder).lte(date); } else { - finalQuery.must(QueryBuilders.rangeQuery(nameValue[0]).lte(date)); + finalQuery.must(QueryBuilders.rangeQuery(nameValue[0].trim()).lte(date)); } } else if (nameValue[0].startsWith(DSM)) { - QueryBuilder tmpBuilder = findQueryBuilderForFieldName(finalQuery, nameValue[0]); + QueryBuilder tmpBuilder = findQueryBuilderForFieldName(finalQuery, nameValue[0].trim()); if (tmpBuilder != null) { ((RangeQueryBuilder) tmpBuilder).lte(userEntered); } else { - finalQuery.must(QueryBuilders.rangeQuery(nameValue[0]).lte(userEntered)); + finalQuery.must(QueryBuilders.rangeQuery(nameValue[0].trim()).lte(userEntered)); } } else if (nameValue[0].startsWith(DATA)) { String[] dataParam = nameValue[0].split("\\."); - QueryBuilder tmpBuilder = findQueryBuilderForFieldName(finalQuery, dataParam[1]); + QueryBuilder tmpBuilder = findQueryBuilderForFieldName(finalQuery, dataParam[1].trim()); try { long date = SystemUtil.getLongFromString(userEntered); if (tmpBuilder != null) { ((RangeQueryBuilder) tmpBuilder).lte(date); } else { - finalQuery.must(QueryBuilders.rangeQuery(dataParam[1]).lte(date)); + finalQuery.must(QueryBuilders.rangeQuery(dataParam[1].trim()).lte(date)); } } catch (ParseException e) { logger.error("range was not date. user entered: " + userEntered); } } else if (nameValue[0].startsWith(INVITATIONS)) { String[] invitationParam = nameValue[0].split("\\."); - QueryBuilder tmpBuilder = findQueryBuilderForFieldName(finalQuery, invitationParam[1]); + QueryBuilder tmpBuilder = findQueryBuilderForFieldName(finalQuery, invitationParam[1].trim()); try { long date = SystemUtil.getLongFromString(userEntered); if (tmpBuilder != null) { ((RangeQueryBuilder) tmpBuilder).lte(date); } else { - finalQuery.must(QueryBuilders.rangeQuery(INVITATIONS + DBConstants.ALIAS_DELIMITER + invitationParam[1]).lte(date)); + finalQuery.must(QueryBuilders.rangeQuery(INVITATIONS + DBConstants.ALIAS_DELIMITER + invitationParam[1].trim()).lte(date)); } } catch (ParseException e) { logger.error("range was not date. user entered: " + userEntered); } } else { String[] surveyParam = nameValue[0].split("\\."); - if (CREATED_AT.equals(surveyParam[1]) || COMPLETED_AT.equals(surveyParam[1]) || LAST_UPDATED.equals(surveyParam[1])) { - QueryBuilder tmpBuilder = findQueryBuilderForFieldName(finalQuery, surveyParam[1]); + if (CREATED_AT.equals(surveyParam[1].trim()) || COMPLETED_AT.equals(surveyParam[1].trim()) || LAST_UPDATED.equals(surveyParam[1].trim())) { + QueryBuilder tmpBuilder = findQueryBuilderForFieldName(finalQuery, surveyParam[1].trim()); try { long date = SystemUtil.getLongFromString(userEntered); if (tmpBuilder != null) { ((RangeQueryBuilder) tmpBuilder).gte(date); } else { tmpBuilder = new BoolQueryBuilder(); - ((BoolQueryBuilder) tmpBuilder).must(QueryBuilders.rangeQuery(ACTIVITIES + DBConstants.ALIAS_DELIMITER + surveyParam[1]).lte(date)); + ((BoolQueryBuilder) tmpBuilder).must(QueryBuilders.rangeQuery(ACTIVITIES + DBConstants.ALIAS_DELIMITER + surveyParam[1].trim()).lte(date)); BoolQueryBuilder activityAnswer = new BoolQueryBuilder(); activityAnswer.must(tmpBuilder); - activityAnswer.must(QueryBuilders.matchQuery(ACTIVITIES + DBConstants.ALIAS_DELIMITER + ACTIVITY_CODE, surveyParam[0]).operator(Operator.AND)); + activityAnswer.must(QueryBuilders.matchQuery(ACTIVITIES + DBConstants.ALIAS_DELIMITER + ACTIVITY_CODE, surveyParam[0].trim()).operator(Operator.AND)); NestedQueryBuilder query = QueryBuilders.nestedQuery(ACTIVITIES, activityAnswer, ScoreMode.Avg); finalQuery.must(query); } @@ -1015,7 +1021,13 @@ public static AbstractQueryBuilder> createESQu } } else if (StringUtils.isNumeric(userEntered)) { // separately process expressions with numbers NestedQueryBuilder query = addRangeLimitForNumber( - Filter.SMALLER_EQUALS, surveyParam[1], userEntered, finalQuery, queryPartsMap); + Filter.SMALLER_EQUALS, surveyParam[1].trim(), userEntered, finalQuery, queryPartsMap, ACTIVITIES_QUESTIONS_ANSWER_ANSWER); + if (query != null) { + parentNestedOfRangeBuilderOfNumbers = query; + } + } else if (userEntered.trim().matches("\\d{4}-\\d{1,2}-\\d{1,2}")) { // separately process expressions with numbers + NestedQueryBuilder query = addRangeLimitForNumber( + Filter.SMALLER_EQUALS, surveyParam[1].trim(), userEntered, finalQuery, queryPartsMap, ACTIVITIES_QUESTIONS_ANSWER_DATE); if (query != null) { parentNestedOfRangeBuilderOfNumbers = query; } @@ -1040,9 +1052,9 @@ public static AbstractQueryBuilder> createESQu finalQuery.must(existsQuery); } else { String[] surveyParam = nameValue[0].split("\\."); - if (CREATED_AT.equals(surveyParam[1]) || COMPLETED_AT.equals(surveyParam[1]) || LAST_UPDATED.equals(surveyParam[1]) || STATUS.equals(surveyParam[1])) { + if (CREATED_AT.equals(surveyParam[1].trim()) || COMPLETED_AT.equals(surveyParam[1].trim()) || LAST_UPDATED.equals(surveyParam[1].trim()) || STATUS.equals(surveyParam[1].trim())) { BoolQueryBuilder activityAnswer = new BoolQueryBuilder(); - ExistsQueryBuilder existsQuery = new ExistsQueryBuilder(ACTIVITIES + DBConstants.ALIAS_DELIMITER + surveyParam[1]); + ExistsQueryBuilder existsQuery = new ExistsQueryBuilder(ACTIVITIES + DBConstants.ALIAS_DELIMITER + surveyParam[1].trim()); activityAnswer.must(existsQuery); activityAnswer.must(QueryBuilders.matchQuery(ACTIVITIES + DBConstants.ALIAS_DELIMITER + ACTIVITY_CODE, surveyParam[0].trim())); NestedQueryBuilder queryActivityAnswer = QueryBuilders.nestedQuery(ACTIVITIES, activityAnswer, ScoreMode.Avg); @@ -1063,9 +1075,8 @@ public static AbstractQueryBuilder> createESQu queryBuilder.must(queryActivityAnswer); NestedQueryBuilder query = QueryBuilders.nestedQuery(ACTIVITIES, queryBuilder, ScoreMode.Avg); finalQuery.must(query); - finalQuery = processIsNotNullForRangeOfNumbers( - surveyParam[1], activityAnswer, finalQuery, queryPartsMap, parentNestedOfRangeBuilderOfNumbers); + surveyParam[1].trim(), activityAnswer, finalQuery, queryPartsMap, parentNestedOfRangeBuilderOfNumbers); } } } else { @@ -1099,17 +1110,20 @@ private static QueryBuilder findQueryBuilderForFieldName(BoolQueryBuilder finalQ /** * Find a query which was added to finalQuery for the same [field_name] - * @param tmpFilters parent block must() of a final query - * @param fieldName name of a field which queried - * @param queryPartsMap map with queries paths which was added for field names + * + * @param tmpFilters parent block must() of a final query + * @param fieldName name of a field which queried + * @param queryPartsMap map with queries paths which was added for field names */ - private static QueryBuilder findQueryBuilder(List tmpFilters, @NonNull String fieldName, Map queryPartsMap) { QueryBuilder tmpBuilder = null; + private static QueryBuilder findQueryBuilder(List tmpFilters, @NonNull String fieldName, Map queryPartsMap) { + QueryBuilder tmpBuilder = null; if (!tmpFilters.isEmpty()) { for (Iterator iterator = tmpFilters.iterator(); iterator.hasNext() && tmpBuilder == null; ) { QueryBuilder builder = iterator.next(); if (builder instanceof RangeQueryBuilder && (((RangeQueryBuilder) builder).fieldName().equals(fieldName) || - queryPartsMap != null && fieldName.equals(queryPartsMap.get(((RangeQueryBuilder) builder).fieldName())))) { tmpBuilder = builder; + queryPartsMap != null && fieldName.equals(queryPartsMap.get(((RangeQueryBuilder) builder).fieldName())))) { + tmpBuilder = builder; } else if (builder instanceof NestedQueryBuilder) { tmpBuilder = findQueryBuilder(((BoolQueryBuilder) ((NestedQueryBuilder) builder).query()).must(), fieldName, queryPartsMap); } else { @@ -1166,32 +1180,32 @@ private static QueryBuilder findQueryBuilderForFieldName(BoolQueryBuilder finalQ * a same fieldName (for example related tp `SELF_CURRENT_AGE`) * @return NestedQueryBuilder - built query containing the numeric Range */ - private static NestedQueryBuilder addRangeLimitForNumber( + private static NestedQueryBuilder addRangeLimitForNumber ( String operatorType, String fieldName, String userEnteredValue, BoolQueryBuilder finalQuery, - Map queryPartsMap) { + Map queryPartsMap, + String questionAnswerType) { QueryBuilder tmpBuilder = findQueryBuilderForFieldName(finalQuery, fieldName, queryPartsMap); if (tmpBuilder != null) { if (operatorType.equals(Filter.LARGER_EQUALS)) { - ((RangeQueryBuilder)tmpBuilder).gte(userEnteredValue); + ((RangeQueryBuilder) tmpBuilder).gte(userEnteredValue); } else { - ((RangeQueryBuilder)tmpBuilder).lte(userEnteredValue); + ((RangeQueryBuilder) tmpBuilder).lte(userEnteredValue); } - } - else { + } else { tmpBuilder = new BoolQueryBuilder(); - RangeQueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery(ACTIVITIES_QUESTIONS_ANSWER_ANSWER); + RangeQueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery(questionAnswerType); if (operatorType.equals(Filter.LARGER_EQUALS)) { - ((BoolQueryBuilder)tmpBuilder).must(rangeQueryBuilder.gte(userEnteredValue)); + ((BoolQueryBuilder) tmpBuilder).must(rangeQueryBuilder.gte(userEnteredValue)); } else { - ((BoolQueryBuilder)tmpBuilder).must(rangeQueryBuilder.lte(userEnteredValue)); + ((BoolQueryBuilder) tmpBuilder).must(rangeQueryBuilder.lte(userEnteredValue)); } NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery(ACTIVITIES, tmpBuilder, ScoreMode.Avg); finalQuery.must(nestedQueryBuilder); - queryPartsMap.put(ACTIVITIES_QUESTIONS_ANSWER_ANSWER, fieldName); + queryPartsMap.put(questionAnswerType, fieldName); return nestedQueryBuilder; } return null; @@ -1323,7 +1337,7 @@ private static void createQuery(@NonNull BoolQueryBuilder finalQuery, @NonNull S //set endDate to midnight of that date String endDate = userEntered + END_OF_DAY; long end = SystemUtil.getLongFromDetailDateString(endDate); - rangeQueryBuilder(finalQuery, nameValue[0], start, end, must); + rangeQueryBuilder(finalQuery, nameValue[0].trim(), start, end, must); } catch (ParseException e) { valueQueryBuilder(finalQuery, nameValue[0].trim(), userEntered, wildCard, must); } @@ -1337,7 +1351,7 @@ private static void createQuery(@NonNull BoolQueryBuilder finalQuery, @NonNull S //set endDate to midnight of that date String endDate = userEntered + END_OF_DAY; long end = SystemUtil.getLongFromDetailDateString(endDate); - rangeQueryBuilder(finalQuery, dataParam[1], start, end, must); + rangeQueryBuilder(finalQuery, dataParam[1].trim(), start, end, must); } catch (ParseException e) { //was no date string so go for normal text mustOrSearch(finalQuery, dataParam[1].trim(), userEntered, wildCard, must); @@ -1354,7 +1368,7 @@ private static void createQuery(@NonNull BoolQueryBuilder finalQuery, @NonNull S //set endDate to midnight of that date String endDate = userEntered + END_OF_DAY; long end = SystemUtil.getLongFromDetailDateString(endDate); - rangeQueryBuilder(queryBuilder, INVITATIONS + DBConstants.ALIAS_DELIMITER + invitationParam[1], start, end, must); + rangeQueryBuilder(queryBuilder, INVITATIONS + DBConstants.ALIAS_DELIMITER + invitationParam[1].trim(), start, end, must); } catch (Exception e) { if (wildCard) { if (must) { @@ -1380,19 +1394,19 @@ private static void createQuery(@NonNull BoolQueryBuilder finalQuery, @NonNull S BoolQueryBuilder queryBuilder = new BoolQueryBuilder(); boolean alreadyAdded = false; - if (CREATED_AT.equals(surveyParam[1]) || COMPLETED_AT.equals(surveyParam[1]) || LAST_UPDATED.equals(surveyParam[1])) { + if (CREATED_AT.equals(surveyParam[1].trim()) || COMPLETED_AT.equals(surveyParam[1].trim()) || LAST_UPDATED.equals(surveyParam[1].trim())) { try { //activity dates long start = SystemUtil.getLongFromString(userEntered); //set endDate to midnight of that date String endDate = userEntered + END_OF_DAY; long end = SystemUtil.getLongFromDetailDateString(endDate); - rangeQueryBuilder(queryBuilder, ACTIVITIES + DBConstants.ALIAS_DELIMITER + surveyParam[1], start, end, must); + rangeQueryBuilder(queryBuilder, ACTIVITIES + DBConstants.ALIAS_DELIMITER + surveyParam[1].trim(), start, end, must); } catch (ParseException e) { //activity status valueQueryBuilder(queryBuilder, ACTIVITIES + DBConstants.ALIAS_DELIMITER + surveyParam[1].trim(), userEntered, wildCard, must); } - } else if (STATUS.equals(surveyParam[1])) { + } else if (STATUS.equals(surveyParam[1].trim())) { if (wildCard) { if (must) { queryBuilder.must(QueryBuilders.wildcardQuery(ACTIVITIES + DBConstants.ALIAS_DELIMITER + surveyParam[1].trim(), userEntered + "*")); @@ -1409,7 +1423,7 @@ private static void createQuery(@NonNull BoolQueryBuilder finalQuery, @NonNull S } } else { //activity user entered - activityAnswer.must(QueryBuilders.matchQuery(ACTIVITIES_QUESTIONS_ANSWER_STABLE_ID, surveyParam[1])); + activityAnswer.must(QueryBuilders.matchQuery(ACTIVITIES_QUESTIONS_ANSWER_STABLE_ID, surveyParam[1].trim())); try { SystemUtil.getLongFromString(userEntered); activityAnswer.must(QueryBuilders.matchQuery(ACTIVITIES_QUESTIONS_ANSWER_DATE, userEntered)); @@ -1445,7 +1459,7 @@ private static void createQuery(@NonNull BoolQueryBuilder finalQuery, @NonNull S } } if (!alreadyAdded) { - queryBuilder.must(QueryBuilders.matchQuery(ACTIVITIES + DBConstants.ALIAS_DELIMITER + ACTIVITY_CODE, surveyParam[0]).operator(Operator.AND)); + queryBuilder.must(QueryBuilders.matchQuery(ACTIVITIES + DBConstants.ALIAS_DELIMITER + ACTIVITY_CODE, surveyParam[0].trim()).operator(Operator.AND)); NestedQueryBuilder query = QueryBuilders.nestedQuery(ACTIVITIES, queryBuilder, ScoreMode.Avg); finalQuery.must(query); } @@ -1556,9 +1570,9 @@ private static boolean isFieldNested(String[] fieldsArray) { private static void rangeQueryBuilder(@NonNull BoolQueryBuilder finalQuery, @NonNull String name, long start, long end, boolean must) { if (must) { - finalQuery.must(QueryBuilders.rangeQuery(name).gte(start).lte(end)); + finalQuery.must(QueryBuilders.rangeQuery(name.trim()).gte(start).lte(end)); } else { - finalQuery.should(QueryBuilders.rangeQuery(name).gte(start).lte(end)); + finalQuery.should(QueryBuilders.rangeQuery(name.trim()).gte(start).lte(end)); } } From 4df78d7b6cac8175d5dce4256283882b7f308bab Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 19 Jan 2022 13:29:10 +0400 Subject: [PATCH 326/405] DDP-7443 | start creating abstractions and concrete implmenetations for upsert mechanism with the help of painless cript --- .../export/ElasticDataExportAdapter.java | 1 + .../export/painless/BaseUpsertPainless.java | 28 +++++++++++++++++ .../export/painless/ParamsGenerator.java | 28 +++++++++++++++++ .../dsm/model/elastic/UtilTest.java | 2 +- .../export/generate/SourceGeneratorTest.java | 31 +------------------ .../export/painless/ParamsGeneratorTest.java | 26 ++++++++++++++++ 6 files changed, 85 insertions(+), 31 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/BaseUpsertPainless.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGenerator.java create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGeneratorTest.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticDataExportAdapter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticDataExportAdapter.java index c0a27d49f..8d6fe81c5 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticDataExportAdapter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticDataExportAdapter.java @@ -5,6 +5,7 @@ import org.broadinstitute.dsm.model.elastic.Util; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.script.Script; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/BaseUpsertPainless.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/BaseUpsertPainless.java new file mode 100644 index 000000000..f10f4910d --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/BaseUpsertPainless.java @@ -0,0 +1,28 @@ +package org.broadinstitute.dsm.model.elastic.export.painless; + +import org.broadinstitute.dsm.model.elastic.export.BaseExporter; +import org.broadinstitute.dsm.model.elastic.export.Exportable; +import org.broadinstitute.dsm.model.elastic.export.RequestPayload; + +import java.util.Map; + +public abstract class BaseUpsertPainless implements Exportable { + + protected String script; + + + private Map source; + private RequestPayload requestPayload; // index, docId + + public BaseUpsertPainless(String script, Object source, RequestPayload requestPayload) { + this.script = script; +// this.source = source; + this.requestPayload = requestPayload; + } + + @Override + public void export() { + + } + +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGenerator.java new file mode 100644 index 000000000..8e16575c0 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGenerator.java @@ -0,0 +1,28 @@ +package org.broadinstitute.dsm.model.elastic.export.painless; + +import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.model.elastic.export.generate.Generator; +import org.broadinstitute.dsm.statics.ESObjectConstants; + +import java.util.HashMap; +import java.util.Map; + +public class ParamsGenerator implements Generator { + + private Object source; + private String realm; + + public ParamsGenerator(Object source, String realm) { + this.source = source; + this.realm = realm; + } + + @Override + public Map generate() { + String propertyName = Util.capitalCamelCaseToLowerCamelCase(source.getClass().getSimpleName()); + Map fieldsMap = Util.transformObjectToMap(source, realm); + return new HashMap<>(Map.of("params", + new HashMap<>(Map.of(ESObjectConstants.DSM, + new HashMap<>(Map.of(propertyName, fieldsMap)))))); + } +} diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java index f1663a719..f4e643405 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java @@ -45,7 +45,7 @@ public void transformObjectToMap() { "ptNotes", true, true, "additionalValuesJson", 1934283746283L); Map transformedObject = Util.transformObjectToMap(participant, "angio"); - assertEquals("1", transformedObject.get("participantId")); + assertEquals(1L, transformedObject.get("participantId")); assertEquals("QWERTY", transformedObject.get("ddpParticipantId")); assertEquals("2020-10-28", transformedObject.get("created")); assertEquals(true, transformedObject.get("minimalMr")); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorTest.java index bc4d5f5bb..095b6479a 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorTest.java @@ -61,23 +61,7 @@ public void generateFromJson() { maybeDdpInstance.ifPresentOrElse(m -> Assert.assertEquals("TEST", m.get(Util.underscoresToCamelCase("DDP_INSTANCE"))), Assert::fail); } - @Test - public void generateFromMultipleNameValues() { - final String externalOrderNumber = "externalOrderNumber"; - final String bspCollaboratorSampleId = "bspCollaboratorSampleId"; - List nameValues = Arrays.asList(new NameValue(externalOrderNumber, 12), new NameValue(bspCollaboratorSampleId, 55)); - TestGeneratorPayload testGeneratorPayload = new TestGeneratorPayload(nameValues, 1); - SourceGenerator sourceGenerator = new SourceGenerator(new ValueParser(), testGeneratorPayload); - Map generatedMap = sourceGenerator.generate(); - // dsm.kitRequestShipping.[{}] - List> kitRequestShippings = (List>) ((Map) generatedMap.get("dsm")).get("kitRequestShipping"); - for (Map kitRequestShipping : kitRequestShippings) { - Object externalOrderNumberVal = kitRequestShipping.get(externalOrderNumber); - Object collaboratorSampleIdVal = kitRequestShipping.get(bspCollaboratorSampleId); - Assert.assertEquals("12", externalOrderNumberVal); - Assert.assertEquals("55", collaboratorSampleIdVal); - } - } + private static class TestSourceGenerator extends CollectionSourceGenerator { @@ -91,19 +75,6 @@ protected DBElement getDBElement() { } } - private class TestGeneratorPayload extends GeneratorPayload { - - List nameValue; - public TestGeneratorPayload(List nameValues, int id) { - this.nameValue = nameValues; - this.recordId = id; - } - - - public List getNameValue() { - return nameValue; - } - } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGeneratorTest.java new file mode 100644 index 000000000..2bcbbaa33 --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGeneratorTest.java @@ -0,0 +1,26 @@ +package org.broadinstitute.dsm.model.elastic.export.painless; + +import org.broadinstitute.dsm.db.KitRequestShipping; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Map; + +public class ParamsGeneratorTest { + + @Test + public void generate() { + KitRequestShipping kitRequestShipping = new KitRequestShipping(1L, 2L, "easyPostIdValue", "easyPostAddressIdValue", true, "msg"); + ParamsGenerator paramsGenerator = new ParamsGenerator(kitRequestShipping, ""); + Map paramsMap = paramsGenerator.generate(); + Map params = (Map ) paramsMap.get("params"); + Map dsm = (Map ) params.get("dsm"); + Map kitRequestShippingObj = (Map ) dsm.get("kitRequestShipping"); + Assert.assertEquals(1L, kitRequestShippingObj.get("dsmKitRequestId")); + Assert.assertEquals(2L, kitRequestShippingObj.get("dsmKitId")); + Assert.assertEquals("easyPostIdValue", kitRequestShippingObj.get("easypostToId")); + Assert.assertEquals(true, kitRequestShippingObj.get("error")); + Assert.assertEquals("msg", kitRequestShippingObj.get("message")); + } + +} \ No newline at end of file From 8554fde636081ca8dbea91973ec4f682518b5567 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 19 Jan 2022 17:07:22 +0400 Subject: [PATCH 327/405] DDP-7443 | create concrete classes of baseupsertpainles abstraction --- .../export/ElasticDataExportAdapter.java | 3 +- .../model/elastic/export/RequestPayload.java | 10 +++--- .../elastic/export/generate/Generator.java | 4 +++ .../export/painless/BaseUpsertPainless.java | 31 +++++++++++++------ .../elastic/export/painless/NestedUpsert.java | 23 ++++++++++++++ .../export/painless/ParamsGenerator.java | 11 +++++-- .../elastic/export/painless/SingleUpsert.java | 20 ++++++++++++ .../export/painless/ParamsGeneratorTest.java | 2 +- 8 files changed, 84 insertions(+), 20 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedUpsert.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/SingleUpsert.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticDataExportAdapter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticDataExportAdapter.java index 8d6fe81c5..043009b17 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticDataExportAdapter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticDataExportAdapter.java @@ -5,7 +5,6 @@ import org.broadinstitute.dsm.model.elastic.Util; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.script.Script; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -16,7 +15,7 @@ public class ElasticDataExportAdapter extends BaseExporter { @Override public void export() { logger.info("initialize exporting data to ES"); - UpdateRequest updateRequest = new UpdateRequest(requestPayload.getIndex(), Util.DOC, requestPayload.getId()) + UpdateRequest updateRequest = new UpdateRequest(requestPayload.getIndex(), Util.DOC, requestPayload.getDocId()) .doc(source) .retryOnConflict(5); try { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/RequestPayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/RequestPayload.java index d63620fd4..18c9b4769 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/RequestPayload.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/RequestPayload.java @@ -5,11 +5,11 @@ public class RequestPayload { private String index; - private String id; + private String docId; - public RequestPayload(String index, String id) { + public RequestPayload(String index, String docId) { this.index = index; - this.id = id; + this.docId = docId; } public RequestPayload(String index) { @@ -20,7 +20,7 @@ public String getIndex() { return index; } - public String getId() { - return id; + public String getDocId() { + return docId; } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Generator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Generator.java index 01cc63886..5521dc35c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Generator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Generator.java @@ -4,4 +4,8 @@ public interface Generator { Map generate(); + + default String getPropertyName() { + throw new UnsupportedOperationException(); + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/BaseUpsertPainless.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/BaseUpsertPainless.java index f10f4910d..aae22a401 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/BaseUpsertPainless.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/BaseUpsertPainless.java @@ -1,28 +1,41 @@ package org.broadinstitute.dsm.model.elastic.export.painless; -import org.broadinstitute.dsm.model.elastic.export.BaseExporter; +import java.io.IOException; + +import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.Exportable; import org.broadinstitute.dsm.model.elastic.export.RequestPayload; - -import java.util.Map; +import org.broadinstitute.dsm.model.elastic.export.generate.Generator; +import org.broadinstitute.dsm.util.ElasticSearchUtil; +import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptType; public abstract class BaseUpsertPainless implements Exportable { protected String script; - - - private Map source; + private Generator generator; private RequestPayload requestPayload; // index, docId - public BaseUpsertPainless(String script, Object source, RequestPayload requestPayload) { + public BaseUpsertPainless(String script, Generator generator, RequestPayload requestPayload) { this.script = script; -// this.source = source; + this.generator = generator; this.requestPayload = requestPayload; } @Override public void export() { - + RestHighLevelClient clientInstance = ElasticSearchUtil.getClientInstance(); + UpdateRequest updateRequest = new UpdateRequest(requestPayload.getIndex(), Util.DOC, requestPayload.getDocId()); + Script painless = new Script(ScriptType.INLINE, "painless", script, generator.generate()); + updateRequest.script(painless); + try { + clientInstance.update(updateRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + throw new RuntimeException("Error occurred while exporting data to ES", e); + } } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedUpsert.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedUpsert.java new file mode 100644 index 000000000..fa7f62d1f --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedUpsert.java @@ -0,0 +1,23 @@ +package org.broadinstitute.dsm.model.elastic.export.painless; + +import org.broadinstitute.dsm.model.elastic.export.RequestPayload; +import org.broadinstitute.dsm.model.elastic.export.generate.Generator; + +public class NestedUpsert extends BaseUpsertPainless { + + private final static String SCRIPT = "if (ctx._source.dsm.#propertyName == null) {ctx._source.dsm.#propertyName = [params.dsm" + + ".#propertyName]} " + + "else {def targets = ctx._source.dsm.#propertyName.findAll(obj -> obj.#uniqueId == params.dsm.#propertyName.#uniqueId);" + + " " + + "for(target in targets) { for (entry in params.dsm.#propertyName.entrySet()) { target.put(entry.getKey(), entry.getValue()) } " + + "}}"; + + public NestedUpsert(Generator generator, + RequestPayload requestPayload) { + super(processScript(), generator, requestPayload); + } + + private static String processScript() { + return null; + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGenerator.java index 8e16575c0..035a855e7 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGenerator.java @@ -9,6 +9,7 @@ public class ParamsGenerator implements Generator { + static final String PARAMS = "params"; private Object source; private String realm; @@ -19,10 +20,14 @@ public ParamsGenerator(Object source, String realm) { @Override public Map generate() { - String propertyName = Util.capitalCamelCaseToLowerCamelCase(source.getClass().getSimpleName()); Map fieldsMap = Util.transformObjectToMap(source, realm); - return new HashMap<>(Map.of("params", + return new HashMap<>(Map.of(PARAMS, new HashMap<>(Map.of(ESObjectConstants.DSM, - new HashMap<>(Map.of(propertyName, fieldsMap)))))); + new HashMap<>(Map.of(getPropertyName(), fieldsMap)))))); + } + + @Override + public String getPropertyName() { + return Util.capitalCamelCaseToLowerCamelCase(source.getClass().getSimpleName()); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/SingleUpsert.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/SingleUpsert.java new file mode 100644 index 000000000..b03c010e9 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/SingleUpsert.java @@ -0,0 +1,20 @@ +package org.broadinstitute.dsm.model.elastic.export.painless; + +import org.broadinstitute.dsm.model.elastic.export.RequestPayload; +import org.broadinstitute.dsm.model.elastic.export.generate.Generator; + +public class SingleUpsert extends BaseUpsertPainless { + + private final static String SCRIPT = "" + + "if (ctx._source.dsm.#propertyName == null) " + + "{ctx._source.dsm.#propertyName = params.dsm.#propertyName} " + + "else {" + + "for (entry in params.dsm.#propertyName.entrySet()) " + + "{ ctx._source.dsm.#propertyName.put(entry.getKey(), entry.getValue()) }" + + "}"; + + public SingleUpsert(Generator generator, + RequestPayload requestPayload) { + super(SCRIPT, generator, requestPayload); + } +} diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGeneratorTest.java index 2bcbbaa33..f9f648460 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGeneratorTest.java @@ -13,7 +13,7 @@ public void generate() { KitRequestShipping kitRequestShipping = new KitRequestShipping(1L, 2L, "easyPostIdValue", "easyPostAddressIdValue", true, "msg"); ParamsGenerator paramsGenerator = new ParamsGenerator(kitRequestShipping, ""); Map paramsMap = paramsGenerator.generate(); - Map params = (Map ) paramsMap.get("params"); + Map params = (Map ) paramsMap.get(ParamsGenerator.PARAMS); Map dsm = (Map ) params.get("dsm"); Map kitRequestShippingObj = (Map ) dsm.get("kitRequestShipping"); Assert.assertEquals(1L, kitRequestShippingObj.get("dsmKitRequestId")); From 73001901d577231c207a2f608541d75049a97d55 Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 19 Jan 2022 18:07:13 +0400 Subject: [PATCH 328/405] DDP-7443 | implement ScriptBuilder and two child classes which will create scripts for single and nested types --- .../dsm/db/KitRequestShipping.java | 6 +++ .../dsm/model/elastic/Util.java | 6 +++ .../export/generate/BaseGenerator.java | 3 +- .../elastic/export/generate/Generator.java | 4 ++ .../export/painless/NestedScriptBuilder.java | 22 ++++++++++ .../elastic/export/painless/NestedUpsert.java | 23 ---------- .../export/painless/ParamsGenerator.java | 5 +++ .../export/painless/ScriptBuilder.java | 5 +++ .../export/painless/SingleScriptBuilder.java | 19 ++++++++ .../elastic/export/painless/SingleUpsert.java | 20 --------- ...psertPainless.java => UpsertPainless.java} | 10 ++--- .../export/painless/ScriptBuilderTest.java | 43 +++++++++++++++++++ 12 files changed, 116 insertions(+), 50 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedScriptBuilder.java delete mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedUpsert.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/ScriptBuilder.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/SingleScriptBuilder.java delete mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/SingleUpsert.java rename src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/{BaseUpsertPainless.java => UpsertPainless.java} (82%) create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/export/painless/ScriptBuilderTest.java diff --git a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java index 308257664..72ede957b 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java @@ -18,6 +18,9 @@ import org.broadinstitute.dsm.model.*; import org.broadinstitute.dsm.model.ddp.DDPParticipant; import org.broadinstitute.dsm.model.ddp.KitDetail; +import org.broadinstitute.dsm.model.elastic.export.generate.Generator; +import org.broadinstitute.dsm.model.elastic.export.painless.ParamsGenerator; +import org.broadinstitute.dsm.model.elastic.export.painless.UpsertPainless; import org.broadinstitute.dsm.statics.ApplicationConfigConstants; import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.statics.QueryExtension; @@ -868,6 +871,9 @@ public static String writeRequest(@NonNull String instanceId, @NonNull String dd throw new RuntimeException("Error adding kit request w/ ddpKitRequestId " + ddpKitRequestId, results.resultException); } + new ParamsGenerator("", ) + new UpsertPainless() + logger.info("Added kitRequest w/ ddpKitRequestId " + ddpKitRequestId); return (String) results.resultValue; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 8601f98f3..e83969728 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -13,6 +13,7 @@ import org.broadinstitute.dsm.db.*; import org.broadinstitute.dsm.db.structure.ColumnName; import org.broadinstitute.dsm.db.structure.DBElement; +import org.broadinstitute.dsm.db.structure.TableName; import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; import org.broadinstitute.dsm.model.elastic.export.parse.DynamicFieldsParser; import org.broadinstitute.dsm.model.elastic.export.parse.ValueParser; @@ -87,6 +88,11 @@ private static String handleAllUppercase(String word) { return CAMEL_CASE_REGEX.matcher(word).matches() ? word : word.toLowerCase(); } + public static String getPrimaryKeyFromClass(Class clazz) { + TableName tableName = Objects.requireNonNull(clazz.getAnnotation(TableName.class)); + return underscoresToCamelCase(tableName.primaryKey()); + } + private static boolean hasNoUnderscores(String[] splittedWords) { return splittedWords.length < 2; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java index 95dc7b35c..90107ef55 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java @@ -132,8 +132,7 @@ public String getPropertyName() { } public String getPrimaryKey() { - TableName tableName = Objects.requireNonNull(propertyClass.getAnnotation(TableName.class)); - return Util.underscoresToCamelCase(tableName.primaryKey()); + return Util.getPrimaryKeyFromClass(propertyClass); } public boolean isCollection() { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Generator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Generator.java index 5521dc35c..718d7cc50 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Generator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Generator.java @@ -8,4 +8,8 @@ public interface Generator { default String getPropertyName() { throw new UnsupportedOperationException(); } + + default String getUniqueIdentifier() { + throw new UnsupportedOperationException(); + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedScriptBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedScriptBuilder.java new file mode 100644 index 000000000..14535d8bc --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedScriptBuilder.java @@ -0,0 +1,22 @@ +package org.broadinstitute.dsm.model.elastic.export.painless; + +public class NestedScriptBuilder implements ScriptBuilder { + + private final static String SCRIPT = "if (ctx._source.dsm.#propertyName == null) {ctx._source.dsm.#propertyName = [params.dsm.#propertyName]} " + + "else {def targets = ctx._source.dsm.#propertyName.findAll(obj -> obj.#uniqueIdentifier == params.dsm.#propertyName.#uniqueIdentifier); " + + "for(target in targets) { for (entry in params.dsm.#propertyName.entrySet()) { target.put(entry.getKey(), entry.getValue()) } }}"; + + private final String propertyName; + private final String uniqueIdentifier; + + public NestedScriptBuilder(String propertyName, String uniqueIdentifier) { + this.propertyName = propertyName; + this.uniqueIdentifier = uniqueIdentifier; + } + + @Override + public String build() { + return SCRIPT.replace("#propertyName", propertyName) + .replace("#uniqueIdentifier", uniqueIdentifier); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedUpsert.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedUpsert.java deleted file mode 100644 index fa7f62d1f..000000000 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedUpsert.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.broadinstitute.dsm.model.elastic.export.painless; - -import org.broadinstitute.dsm.model.elastic.export.RequestPayload; -import org.broadinstitute.dsm.model.elastic.export.generate.Generator; - -public class NestedUpsert extends BaseUpsertPainless { - - private final static String SCRIPT = "if (ctx._source.dsm.#propertyName == null) {ctx._source.dsm.#propertyName = [params.dsm" + - ".#propertyName]} " + - "else {def targets = ctx._source.dsm.#propertyName.findAll(obj -> obj.#uniqueId == params.dsm.#propertyName.#uniqueId);" + - " " + - "for(target in targets) { for (entry in params.dsm.#propertyName.entrySet()) { target.put(entry.getKey(), entry.getValue()) } " + - "}}"; - - public NestedUpsert(Generator generator, - RequestPayload requestPayload) { - super(processScript(), generator, requestPayload); - } - - private static String processScript() { - return null; - } -} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGenerator.java index 035a855e7..59b04d2af 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGenerator.java @@ -30,4 +30,9 @@ public Map generate() { public String getPropertyName() { return Util.capitalCamelCaseToLowerCamelCase(source.getClass().getSimpleName()); } + + @Override + public String getUniqueIdentifier() { + return Util.getPrimaryKeyFromClass(source.getClass()); + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/ScriptBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/ScriptBuilder.java new file mode 100644 index 000000000..4e7fd656b --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/ScriptBuilder.java @@ -0,0 +1,5 @@ +package org.broadinstitute.dsm.model.elastic.export.painless; + +public interface ScriptBuilder { + String build(); +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/SingleScriptBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/SingleScriptBuilder.java new file mode 100644 index 000000000..7cc6b1556 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/SingleScriptBuilder.java @@ -0,0 +1,19 @@ +package org.broadinstitute.dsm.model.elastic.export.painless; + +public class SingleScriptBuilder implements ScriptBuilder { + + private final static String SCRIPT = "if (ctx._source.dsm.#propertyName == null) " + + "{ctx._source.dsm.#propertyName = params.dsm.#propertyName} " + + "else {for (entry in params.dsm.#propertyName.entrySet()) { ctx._source.dsm.#propertyName.put(entry.getKey(), entry.getValue()) }}"; + + private final String propertyName; + + public SingleScriptBuilder(String propertyName) { + this.propertyName = propertyName; + } + + @Override + public String build() { + return SCRIPT.replace("#propertyName", propertyName); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/SingleUpsert.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/SingleUpsert.java deleted file mode 100644 index b03c010e9..000000000 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/SingleUpsert.java +++ /dev/null @@ -1,20 +0,0 @@ -package org.broadinstitute.dsm.model.elastic.export.painless; - -import org.broadinstitute.dsm.model.elastic.export.RequestPayload; -import org.broadinstitute.dsm.model.elastic.export.generate.Generator; - -public class SingleUpsert extends BaseUpsertPainless { - - private final static String SCRIPT = "" + - "if (ctx._source.dsm.#propertyName == null) " + - "{ctx._source.dsm.#propertyName = params.dsm.#propertyName} " + - "else {" + - "for (entry in params.dsm.#propertyName.entrySet()) " + - "{ ctx._source.dsm.#propertyName.put(entry.getKey(), entry.getValue()) }" + - "}"; - - public SingleUpsert(Generator generator, - RequestPayload requestPayload) { - super(SCRIPT, generator, requestPayload); - } -} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/BaseUpsertPainless.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/UpsertPainless.java similarity index 82% rename from src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/BaseUpsertPainless.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/UpsertPainless.java index aae22a401..04fff39d0 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/BaseUpsertPainless.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/UpsertPainless.java @@ -13,23 +13,23 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; -public abstract class BaseUpsertPainless implements Exportable { +public class UpsertPainless implements Exportable { - protected String script; private Generator generator; private RequestPayload requestPayload; // index, docId + private ScriptBuilder scriptBuilder; - public BaseUpsertPainless(String script, Generator generator, RequestPayload requestPayload) { - this.script = script; + public UpsertPainless(Generator generator, RequestPayload requestPayload, ScriptBuilder scriptBuilder) { this.generator = generator; this.requestPayload = requestPayload; + this.scriptBuilder = scriptBuilder; } @Override public void export() { RestHighLevelClient clientInstance = ElasticSearchUtil.getClientInstance(); UpdateRequest updateRequest = new UpdateRequest(requestPayload.getIndex(), Util.DOC, requestPayload.getDocId()); - Script painless = new Script(ScriptType.INLINE, "painless", script, generator.generate()); + Script painless = new Script(ScriptType.INLINE, "painless", scriptBuilder.build(), generator.generate()); updateRequest.script(painless); try { clientInstance.update(updateRequest, RequestOptions.DEFAULT); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/painless/ScriptBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/painless/ScriptBuilderTest.java new file mode 100644 index 000000000..c02ffcb07 --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/painless/ScriptBuilderTest.java @@ -0,0 +1,43 @@ +package org.broadinstitute.dsm.model.elastic.export.painless; + +import org.junit.Assert; +import org.junit.Test; + +public class ScriptBuilderTest { + + @Test + public void buildNested() { + + String propertyName = "kitRequestShipping"; + String uniqueIdentifier = "kitRequestId"; + + ScriptBuilder builder = new NestedScriptBuilder(propertyName, uniqueIdentifier); + String script = builder.build(); + + String expectedScript = "if (ctx._source.dsm.kitRequestShipping == null) {ctx._source.dsm.kitRequestShipping = [params.dsm.kitRequestShipping]} " + + "else {def targets = ctx._source.dsm.kitRequestShipping.findAll(obj -> obj.kitRequestId == params.dsm.kitRequestShipping.kitRequestId); " + + "for(target in targets) { for (entry in params.dsm.kitRequestShipping.entrySet()) { target.put(entry.getKey(), entry.getValue()) } " + + "}}"; + + Assert.assertEquals(expectedScript, script); + } + + @Test + public void buildSingle() { + + String propertyName = "kitRequestShipping"; + + ScriptBuilder builder = new SingleScriptBuilder(propertyName); + String script = builder.build(); + + String expectedScript = "" + + "if (ctx._source.dsm.kitRequestShipping == null) " + + "{ctx._source.dsm.kitRequestShipping = params.dsm.kitRequestShipping} " + + "else {" + + "for (entry in params.dsm.kitRequestShipping.entrySet()) " + + "{ ctx._source.dsm.kitRequestShipping.put(entry.getKey(), entry.getValue()) }" + + "}"; + + Assert.assertEquals(expectedScript, script); + } +} From 8f6c2f16ed74f755d1cd0dec449860aa78eee950 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 20 Jan 2022 10:45:03 +0400 Subject: [PATCH 329/405] DDP-7443 | refactor old tests --- .../dsm/db/KitRequestShipping.java | 4 +- .../export/generate/BaseGenerator.java | 1 + .../elastic/export/generate/Generator.java | 4 - .../export/painless/ParamsGenerator.java | 5 - .../export/parse/DynamicFieldsParser.java | 4 +- .../dsm/model/elastic/UtilTest.java | 41 ++++--- .../model/elastic/export/TestPatchUtil.java | 14 ++- .../export/generate/MappingGeneratorTest.java | 103 ++++++++++-------- .../export/generate/SourceGeneratorTest.java | 62 +++++++---- .../process/CollectionProcessorTest.java | 47 +++++--- 10 files changed, 170 insertions(+), 115 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java index 72ede957b..5da419abf 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java @@ -871,8 +871,8 @@ public static String writeRequest(@NonNull String instanceId, @NonNull String dd throw new RuntimeException("Error adding kit request w/ ddpKitRequestId " + ddpKitRequestId, results.resultException); } - new ParamsGenerator("", ) - new UpsertPainless() +// new ParamsGenerator("", ) +// new UpsertPainless() logger.info("Added kitRequest w/ ddpKitRequestId " + ddpKitRequestId); return (String) results.resultValue; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java index 90107ef55..150585b91 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java @@ -69,6 +69,7 @@ protected String getPrimaryKey() { return getOuterPropertyByAlias().getPrimaryKey(); } + @Override public String getPropertyName() { return getOuterPropertyByAlias().getPropertyName(); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Generator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Generator.java index 718d7cc50..5521dc35c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Generator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/Generator.java @@ -8,8 +8,4 @@ public interface Generator { default String getPropertyName() { throw new UnsupportedOperationException(); } - - default String getUniqueIdentifier() { - throw new UnsupportedOperationException(); - } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGenerator.java index 59b04d2af..035a855e7 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGenerator.java @@ -30,9 +30,4 @@ public Map generate() { public String getPropertyName() { return Util.capitalCamelCaseToLowerCamelCase(source.getClass().getSimpleName()); } - - @Override - public String getUniqueIdentifier() { - return Util.getPrimaryKeyFromClass(source.getClass()); - } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/DynamicFieldsParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/DynamicFieldsParser.java index a43dd09ed..90e0e6039 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/DynamicFieldsParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/DynamicFieldsParser.java @@ -18,7 +18,7 @@ public class DynamicFieldsParser extends BaseParser { public static final String CHECKBOX_TYPE = "CHECKBOX"; public static final String ACTIVITY_STAFF_TYPE = "ACTIVITY_STAFF"; public static final String ACTIVITY_TYPE = "ACTIVITY"; - private String displayType; + protected String displayType; private String possibleValuesJson; private BaseParser parser; public FieldSettingsDao fieldSettingsDao = FieldSettingsDao.of(); @@ -73,7 +73,7 @@ public Object parse(String element) { return parsedValue; } - private void getProperDisplayTypeWithPossibleValues() { + protected void getProperDisplayTypeWithPossibleValues() { Optional fieldSettingsByInstanceNameAndColumnName = fieldSettingsDao.getFieldSettingsByInstanceNameAndColumnName(realm, super.fieldName); if (fieldSettingsByInstanceNameAndColumnName.isPresent()) { diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java index f4e643405..c6ee77072 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java @@ -11,11 +11,34 @@ import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; import org.broadinstitute.dsm.model.FollowUp; +import org.broadinstitute.dsm.model.elastic.export.generate.SourceGeneratorTest; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; public class UtilTest { + @Before + public void setUp() { + class FieldSettingsDaoMock extends FieldSettingsDao { + + @Override + public Optional getFieldSettingsByInstanceNameAndColumnName(String instanceName, String columnName) { + FieldSettingsDto.Builder builder = new FieldSettingsDto.Builder(0); + if ("BOOLEAN_VAL".equals(columnName)) { + builder.withDisplayType("CHECKBOX"); + } else if ("LONG_VAL".equals(columnName)) { + builder.withDisplayType("NUMBER"); + } + return Optional.of(builder.build()); + } + } + + FieldSettingsDaoMock fieldSettingsDaoMock = new FieldSettingsDaoMock(); + + FieldSettingsDao.setInstance(fieldSettingsDaoMock); + } + @Test public void underscoresToCamelCase() { String fieldName = "column_name"; @@ -55,23 +78,7 @@ public void transformObjectToMap() { @Test public void transformJsonToMap() { - class FieldSettingsDaoMock extends FieldSettingsDao { - - @Override - public Optional getFieldSettingsByInstanceNameAndColumnName(String instanceName, String columnName) { - FieldSettingsDto.Builder builder = new FieldSettingsDto.Builder(0); - if ("BOOLEAN_VAL".equals(columnName)) { - builder.withDisplayType("CHECKBOX"); - } else if ("LONG_VAL".equals(columnName)) { - builder.withDisplayType("NUMBER"); - } - return Optional.of(builder.build()); - } - } - - FieldSettingsDaoMock fieldSettingsDaoMock = new FieldSettingsDaoMock(); - - FieldSettingsDao.setInstance(fieldSettingsDaoMock); + setUp(); String json = "{\"DDP_INSTANCE\": \"TEST\", \"DDP_VALUE\": \"VALUE\", \"BOOLEAN_VAL\": \"true\", \"LONG_VAL\": \"5\"}"; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/TestPatchUtil.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/TestPatchUtil.java index a5097575c..985776b04 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/TestPatchUtil.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/TestPatchUtil.java @@ -9,13 +9,25 @@ public class TestPatchUtil extends PatchUtil { public static final String MEDICAL_RECORD_COLUMN = "medical_record_column"; public static final String TISSUE_RECORD_COLUMN = "tissue_record_column"; + public static final String MR_PROBLEM = "mr_problem"; + public static final String DATE_FIELD = "date_field"; + public static final String NUMERIC_FIELD = "numeric_field"; + public static final String ADDITIONAL_VALUES_JSON = "additional_values_json"; public static Map getColumnNameMap() { DBElement medicalRecord = new DBElement("ddp_medical_record", "m", "pr", MEDICAL_RECORD_COLUMN); DBElement tissueRecord = new DBElement("ddp_tissue_record", "t", "pr", TISSUE_RECORD_COLUMN); + DBElement mrProblem = new DBElement("mr_problem", "m", "pr", MR_PROBLEM); + DBElement dateField = new DBElement("date_field", "m", "pr", DATE_FIELD); + DBElement numericField = new DBElement("numeric_field", "m", "pr", NUMERIC_FIELD); + DBElement additionalValuesJson = new DBElement("additional_values_json", "m", "pr", ADDITIONAL_VALUES_JSON); return Map.of( MEDICAL_RECORD_COLUMN, medicalRecord, - TISSUE_RECORD_COLUMN, tissueRecord + TISSUE_RECORD_COLUMN, tissueRecord, + MR_PROBLEM, mrProblem, + DATE_FIELD, dateField, + NUMERIC_FIELD, numericField, + ADDITIONAL_VALUES_JSON, additionalValuesJson ); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java index 7b310d2da..572b1572d 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/MappingGeneratorTest.java @@ -1,18 +1,21 @@ package org.broadinstitute.dsm.model.elastic.export.generate; +import org.broadinstitute.dsm.db.MedicalRecord; import org.broadinstitute.dsm.db.structure.DBElement; +import org.broadinstitute.dsm.db.structure.DbDateConversion; +import org.broadinstitute.dsm.db.structure.SqlDateConverter; import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.model.elastic.export.parse.BaseParser; +import org.broadinstitute.dsm.model.elastic.export.parse.DynamicFieldsParser; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import org.broadinstitute.dsm.model.elastic.export.TestPatchUtil; import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; import org.junit.Assert; import org.junit.Test; -import java.util.HashMap; import java.util.Map; -import java.util.Objects; import static org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator.PROPERTIES; @@ -21,73 +24,64 @@ public class MappingGeneratorTest { @Test public void generateTextType() { - GeneratorPayload generatorPayload = new GeneratorPayload( - new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "value"), - 0 - ); - Map objectMap = TestCollectionMappingGenerator.of(generatorPayload).generate(); + Map objectMap = TestCollectionMappingGenerator.of(getGeneratorPayload(TestPatchUtil.MEDICAL_RECORD_COLUMN, "value" + , 0)).generate(); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), PROPERTIES); - String type = extractDeepestLeveleValue(objectMap); + String type = extractDeepestLeveleValue(objectMap, TestPatchUtil.MEDICAL_RECORD_COLUMN); Assert.assertEquals("text", type); } + public static GeneratorPayload getGeneratorPayload(String columnName, Object value, int recordId) { + return new GeneratorPayload(new NameValue(columnName, value), + recordId) { + @Override + public String getCamelCaseFieldName() { + return Util.underscoresToCamelCase(columnName); + } + }; + } + @Test public void generateTextTypeWithFields() { - GeneratorPayload generatorPayload = new GeneratorPayload( - new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "value"), - 0 - ); - Map objectMap = TestCollectionMappingGenerator.of(generatorPayload).generate(); + Map objectMap = TestCollectionMappingGenerator.of(getGeneratorPayload(TestPatchUtil.MEDICAL_RECORD_COLUMN,"value" + , 0)).generate(); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), PROPERTIES); - String type = extractKeywordType(objectMap); + String type = extractKeywordType(objectMap, TestPatchUtil.MEDICAL_RECORD_COLUMN); Assert.assertEquals("keyword", type); } @Test public void generateBooleanType() { - GeneratorPayload generatorPayload = new GeneratorPayload( - new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "true"), - 0 - ); - Map objectMap = TestCollectionMappingGenerator.of(generatorPayload).generate(); + Map objectMap = TestCollectionMappingGenerator.of(getGeneratorPayload(TestPatchUtil.MR_PROBLEM,true, + 0)).generate(); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), PROPERTIES); - String type = extractDeepestLeveleValue(objectMap); + String type = extractDeepestLeveleValue(objectMap, TestPatchUtil.MR_PROBLEM); Assert.assertEquals("boolean", type); } @Test public void generateDateType() { - GeneratorPayload generatorPayload = new GeneratorPayload( - new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "2021-10-30"), - 0 - ); - Map objectMap = TestCollectionMappingGenerator.of(generatorPayload).generate(); + Map objectMap = + TestCollectionMappingGenerator.of(getGeneratorPayload(TestPatchUtil.DATE_FIELD, "2021-10-30", 0)).generate(); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), PROPERTIES); - String type = extractDeepestLeveleValue(objectMap); + String type = extractDeepestLeveleValue(objectMap, TestPatchUtil.DATE_FIELD); Assert.assertEquals("date", type); } @Test public void generateNestedType() { - GeneratorPayload generatorPayload = new GeneratorPayload( - new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "2021-10-30"), - 100 - ); - Map objectMap = TestCollectionMappingGenerator.of(generatorPayload).generate(); + Map objectMap = + TestCollectionMappingGenerator.of(getGeneratorPayload(TestPatchUtil.DATE_FIELD, "2021-10-30", 100)).generate(); Assert.assertEquals(MappingGenerator.NESTED, getMedicalRecordProperty(objectMap).get(MappingGenerator.TYPE)); } @Test public void generateMapping() { - GeneratorPayload generatorPayload = new GeneratorPayload( - new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "2021-10-30"), - 100 - ); - MappingGenerator generator = TestCollectionMappingGenerator.of(generatorPayload); + MappingGenerator generator = TestCollectionMappingGenerator.of(getGeneratorPayload(TestPatchUtil.DATE_FIELD, "2021-10-30", 100)); Map resultMap = generator.generate(); Map dsmLevelProperty = Map.of(generator.getPropertyName(), Map.of( MappingGenerator.TYPE, MappingGenerator.NESTED, - PROPERTIES, Map.of(Util.underscoresToCamelCase(TestPatchUtil.MEDICAL_RECORD_COLUMN), Map.of(MappingGenerator.TYPE, "date")))); + PROPERTIES, Map.of(Util.underscoresToCamelCase(TestPatchUtil.DATE_FIELD), Map.of(MappingGenerator.TYPE, "date")))); Map dsmLevelProperties = Map.of(PROPERTIES, dsmLevelProperty); Map dsmLevel = Map.of(MappingGenerator.DSM_OBJECT, dsmLevelProperties); Map topLevel = Map.of(PROPERTIES, dsmLevel); @@ -96,29 +90,31 @@ public void generateMapping() { @Test public void parseJson() { - NameValue nameValue = new NameValue("m.additionalValuesJson", "{\"DDP_INSTANCE\": \"TEST\", \"DDP_INSTANCE1\": \"TEST1\"}"); + NameValue nameValue = new NameValue("m.additionalValuesJson", "{\"DDP_INSTANCE\": \"TEST\"}"); GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, 0); - TypeParser parser = new TypeParser(); - MappingGenerator mappingGenerator = new CollectionMappingGenerator(parser, generatorPayload); + DynamicFieldsParser parser = new DynamicFieldsParser(); + parser.setDisplayType("TEXT"); + parser.setParser(new TypeParser()); + parser.setPropertyInfo(new BaseGenerator.PropertyInfo(MedicalRecord.class, true)); + BaseGenerator mappingGenerator = new CollectionMappingGenerator(parser, generatorPayload); Map parseJson = mappingGenerator.parseJson(); - Map additionalValuesJson = (Map)parseJson.get("additionalValuesJson"); + Map additionalValuesJson = (Map)parseJson.get("dynamicFields"); Assert.assertNotNull(additionalValuesJson); Assert.assertEquals(TypeParser.TEXT_KEYWORD_MAPPING,((Map) additionalValuesJson.get(PROPERTIES)).get(Util.underscoresToCamelCase("DDP_INSTANCE"))); - Assert.assertEquals(TypeParser.TEXT_KEYWORD_MAPPING,((Map) additionalValuesJson.get(PROPERTIES)).get(Util.underscoresToCamelCase("DDP_INSTANCE1"))); } - private String extractDeepestLeveleValue(Map objectMap) { + private String extractDeepestLeveleValue(Map objectMap, String field) { return (String) ((Map) ((Map) getMedicalRecordProperty(objectMap) .get(PROPERTIES)) - .get("medicalRecordColumn")) + .get(Util.underscoresToCamelCase(field))) .get("type"); } - private String extractKeywordType(Map objectMap) { + private String extractKeywordType(Map objectMap, String field) { return (String) ((Map) ((Map) @@ -126,7 +122,7 @@ private String extractKeywordType(Map objectMap) { ((Map) getMedicalRecordProperty(objectMap) .get(PROPERTIES)) - .get("medicalRecordColumn")) + .get(Util.underscoresToCamelCase(field))) .get("fields")) .get("keyword")) .get("type"); @@ -140,7 +136,7 @@ private Map getMedicalRecordProperty(Map objectMap) { .get(PROPERTIES)) .get(BaseGenerator.DSM_OBJECT)) .get(PROPERTIES)) - .get("medicalRecords"); + .get("medicalRecord"); } @@ -174,7 +170,18 @@ protected DBElement getDBElement() { } public static TestCollectionMappingGenerator of(GeneratorPayload generatorPayload) { - return new TestCollectionMappingGenerator(new TypeParser(), generatorPayload); + BaseParser typeParser = new TypeParser(); + typeParser.setPropertyInfo(new PropertyInfo(TestPropertyClass.class, true)); + return new TestCollectionMappingGenerator(typeParser, generatorPayload); } } + + public static class TestPropertyClass { + String medicalRecordColumn; + String tissueRecordColumn; + boolean mrProblem; + @DbDateConversion(SqlDateConverter.STRING_DAY) + String dateField; + long numericField; + } } \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorTest.java index 095b6479a..8b00092d7 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/generate/SourceGeneratorTest.java @@ -5,11 +5,16 @@ import java.util.Map; import java.util.Optional; +import org.apache.commons.lang3.StringUtils; +import org.broadinstitute.dsm.db.MedicalRecord; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.model.elastic.export.parse.BaseParser; +import org.broadinstitute.dsm.model.elastic.export.parse.DynamicFieldsParser; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; import org.broadinstitute.dsm.model.elastic.export.TestPatchUtil; +import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; import org.broadinstitute.dsm.model.elastic.export.parse.ValueParser; import org.junit.Test; import org.junit.Assert; @@ -18,47 +23,59 @@ public class SourceGeneratorTest { @Test public void generateCollection() { - NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "value"); - GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, 0); - Generator generator = new TestSourceGenerator(new ValueParser(), generatorPayload); + BaseParser parser = new ValueParser(); + parser.setPropertyInfo(new BaseGenerator.PropertyInfo(MappingGeneratorTest.TestPropertyClass.class, true)); + Generator generator = new TestSourceGenerator(parser, getGeneratorPayload(TestPatchUtil.MEDICAL_RECORD_COLUMN, "value" + , 0)); Map objectMap = generator.generate(); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), SourceGenerator.DSM_OBJECT); - List> medicalRecords = (List) ((Map) objectMap.get(SourceGenerator.DSM_OBJECT)).get("medicalRecords"); + List> medicalRecords = (List) ((Map) objectMap.get(SourceGenerator.DSM_OBJECT)).get("medicalRecord"); Optional> first = medicalRecords.stream() .filter(i -> i.get(Util.underscoresToCamelCase(TestPatchUtil.MEDICAL_RECORD_COLUMN)) != null) .findFirst(); first.ifPresentOrElse(val -> Assert.assertEquals("value", val.get(Util.underscoresToCamelCase(TestPatchUtil.MEDICAL_RECORD_COLUMN))), Assert::fail); } + private GeneratorPayload getGeneratorPayload(String columnName, Object value, int recordId) { + return new GeneratorPayload(new NameValue(columnName, value), + recordId) { + @Override + public String getCamelCaseFieldName() { + return Util.underscoresToCamelCase(columnName); + } + }; + } + @Test public void generateNumeric() { - NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "1"); - GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, 0); - Generator generator = new TestSourceGenerator(new ValueParser(), generatorPayload); + BaseParser parser = new ValueParser(); + parser.setPropertyInfo(new BaseGenerator.PropertyInfo(MappingGeneratorTest.TestPropertyClass.class, true)); + Generator generator = new TestSourceGenerator(parser, getGeneratorPayload(TestPatchUtil.NUMERIC_FIELD, 1, 0)); Map objectMap = generator.generate(); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), SourceGenerator.DSM_OBJECT); - List> medicalRecords = (List) ((Map) objectMap.get(SourceGenerator.DSM_OBJECT)).get("medicalRecords"); + List> medicalRecords = (List) ((Map) objectMap.get(SourceGenerator.DSM_OBJECT)).get("medicalRecord"); Optional> first = medicalRecords.stream() - .filter(i -> i.get(Util.underscoresToCamelCase(TestPatchUtil.MEDICAL_RECORD_COLUMN)) != null) + .filter(i -> i.get(Util.underscoresToCamelCase(TestPatchUtil.NUMERIC_FIELD)) != null) .findFirst(); - first.ifPresentOrElse(val -> Assert.assertEquals(1L, val.get(Util.underscoresToCamelCase(TestPatchUtil.MEDICAL_RECORD_COLUMN))), Assert::fail); + first.ifPresentOrElse(val -> Assert.assertEquals(1L, val.get(Util.underscoresToCamelCase(TestPatchUtil.NUMERIC_FIELD))), Assert::fail); } @Test public void generateFromJson() { - NameValue nameValue = new NameValue(TestPatchUtil.TISSUE_RECORD_COLUMN, "{\"DDP_INSTANCE\": \"TEST\", \"DPP_VALUE\": \"VALUE\"}"); + NameValue nameValue = new NameValue("additional_values_json", "{\"DDP_INSTANCE\": \"TEST\", \"DDP_VALUE\": \"VALUE\"}"); GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, 0); - Generator generator = new TestSourceGenerator(new ValueParser(), generatorPayload); + DynamicFieldsParser dynamicFieldsParser = new TestDynamicFieldsParser(); + dynamicFieldsParser.setParser(new ValueParser()); + dynamicFieldsParser.setPropertyInfo(new BaseGenerator.PropertyInfo(MedicalRecord.class, true)); + Generator generator = new TestSourceGenerator(dynamicFieldsParser, generatorPayload); Map objectMap = generator.generate(); Assert.assertEquals(objectMap.keySet().stream().findFirst().get(), SourceGenerator.DSM_OBJECT); - List> tissueRecords = (List>) ((Map) objectMap - .get(SourceGenerator.DSM_OBJECT)) - .get("tissueRecords"); - Optional> maybeDdpInstance = tissueRecords.stream() - .filter(i -> i.get(Util.underscoresToCamelCase("DDP_INSTANCE")) != null) - .findFirst(); - maybeDdpInstance.ifPresentOrElse(m -> Assert.assertEquals("TEST", m.get(Util.underscoresToCamelCase("DDP_INSTANCE"))), Assert::fail); + List> medicalRecord = + (List>) ((Map) objectMap.get(SourceGenerator.DSM_OBJECT)).get("medicalRecord"); + Map dynamicFields = (Map) medicalRecord.get(0).get("dynamicFields"); + Assert.assertEquals("TEST", dynamicFields.get("ddpInstance")); + Assert.assertEquals("VALUE", dynamicFields.get("ddpValue")); } @@ -75,6 +92,13 @@ protected DBElement getDBElement() { } } + public static class TestDynamicFieldsParser extends DynamicFieldsParser { + + @Override + protected void getProperDisplayTypeWithPossibleValues() { + displayType = StringUtils.EMPTY; + } + } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorTest.java index 8e6ffec82..6fde1ede7 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorTest.java @@ -1,15 +1,22 @@ package org.broadinstitute.dsm.model.elastic.export.process; import com.fasterxml.jackson.databind.ObjectMapper; +import org.broadinstitute.dsm.db.MedicalRecord; import org.broadinstitute.dsm.db.structure.DBElement; import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.elastic.ESDsm; import org.broadinstitute.dsm.model.elastic.export.TestPatchUtil; +import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; +import org.broadinstitute.dsm.model.elastic.export.generate.CollectionSourceGenerator; import org.broadinstitute.dsm.model.elastic.export.generate.Collector; import org.broadinstitute.dsm.model.elastic.export.generate.Generator; import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; +import org.broadinstitute.dsm.model.elastic.export.generate.MappingGeneratorTest; import org.broadinstitute.dsm.model.elastic.export.generate.SourceGenerator; +import org.broadinstitute.dsm.model.elastic.export.parse.BaseParser; +import org.broadinstitute.dsm.model.elastic.export.parse.DynamicFieldsParser; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; +import org.broadinstitute.dsm.model.elastic.export.parse.TypeParser; import org.broadinstitute.dsm.model.elastic.export.parse.ValueParser; import org.broadinstitute.dsm.model.elastic.export.process.CollectionProcessor; import org.junit.Assert; @@ -23,7 +30,7 @@ public class CollectionProcessorTest { @Test public void testProcess() throws IOException { - String propertyName = "medicalRecords"; + String propertyName = "medicalRecord"; double recordId = 5; String oldValue = "mr_old"; String json = String.format("{\"%s\":[{\"id\":%s,\"mr\":\"%s\"}]}", propertyName, recordId, oldValue); @@ -33,10 +40,10 @@ public void testProcess() throws IOException { ESDsm esDsm = objectMapper.readValue(json, ESDsm.class); NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "mr_updated"); - GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, (int)recordId); - - Processor collectionProcessor = new TestCollectionProcessor(esDsm, propertyName, generatorPayload, instance(generatorPayload, - nameValue)); + GeneratorPayload generatorPayload = MappingGeneratorTest.getGeneratorPayload(TestPatchUtil.MEDICAL_RECORD_COLUMN, "mr_updated", 5); + Processor collectionProcessor = new TestCollectionProcessor(esDsm, propertyName, generatorPayload, + instance(generatorPayload, + nameValue, new ValueParser())); List> updatedList = (List>) collectionProcessor.process(); @@ -49,7 +56,7 @@ public void testProcess() throws IOException { @Test public void updateIfExistsOrPut() throws IOException { - String propertyName = "medicalRecords"; + String propertyName = "medicalRecord"; double recordId = 5; String json = String.format("{\"%s\":[{\"id\":%s,\"mr\":\"%s\"}]}", propertyName, recordId, "value");; @@ -58,10 +65,9 @@ public void updateIfExistsOrPut() throws IOException { ESDsm esDsm = objectMapper.readValue(json, ESDsm.class); NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "val"); - GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, 10); - + GeneratorPayload generatorPayload = MappingGeneratorTest.getGeneratorPayload(TestPatchUtil.MEDICAL_RECORD_COLUMN, "val", 10); CollectionProcessor collectionProcessor = new TestCollectionProcessor(esDsm, propertyName, generatorPayload, - instance(generatorPayload, nameValue)); + instance(generatorPayload, nameValue, new ValueParser())); List> updatedList = collectionProcessor.process(); @@ -71,7 +77,7 @@ public void updateIfExistsOrPut() throws IOException { @Test public void updateIfExists() throws IOException { - String propertyName = "medicalRecords"; + String propertyName = "medicalRecord"; double recordId = 5; String json = String.format("{\"%s\":[{\"id\":%s,\"TEST1\":\"%s\", \"TEST2\":\"TEST_VAL2\"}]}", propertyName, recordId, "value");; @@ -79,11 +85,14 @@ public void updateIfExists() throws IOException { ESDsm esDsm = objectMapper.readValue(json, ESDsm.class); - NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "{\"TEST1\":\"TEST_VAL\", \"TEST2\":\"TEST_VAL3\"}"); + NameValue nameValue = new NameValue(TestPatchUtil.ADDITIONAL_VALUES_JSON, "{\"TEST1\":\"TEST_VAL\"}"); GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, 5); - + DynamicFieldsParser parser = new DynamicFieldsParser(); + parser.setDisplayType("TEXT"); + parser.setParser(new ValueParser()); + parser.setPropertyInfo(new BaseGenerator.PropertyInfo(MedicalRecord.class, true)); CollectionProcessor collectionProcessor = new TestCollectionProcessor(esDsm, propertyName, generatorPayload, - instance(generatorPayload, nameValue)); + instance(generatorPayload, nameValue, parser)); List> updatedList = collectionProcessor.process(); @@ -98,10 +107,14 @@ public void updateIfExists() throws IOException { .ifPresentOrElse(m -> m.get("TEST2").equals("TEST_VAL3"), Assert::fail); } - private Collector instance(GeneratorPayload generatorPayload, NameValue nameValue) { -// SourceGenerator sourceGenerator = new SourceGenerator(new ValueParser(), generatorPayload); - SourceGenerator sourceGenerator = null; - sourceGenerator.setDBElement(TestPatchUtil.getColumnNameMap().get(nameValue.getName())); + private Collector instance(GeneratorPayload generatorPayload, NameValue nameValue, BaseParser parser) { + parser.setPropertyInfo(new BaseGenerator.PropertyInfo(MappingGeneratorTest.TestPropertyClass.class, true)); + BaseGenerator sourceGenerator = new CollectionSourceGenerator(parser, generatorPayload) { + @Override + protected DBElement getDBElement() { + return TestPatchUtil.getColumnNameMap().get(nameValue.getName()); + } + }; return sourceGenerator; } From 9cb5da55f2cba0b2ced95f328ff199bcf9414ce8 Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 20 Jan 2022 11:40:46 +0400 Subject: [PATCH 330/405] DDP-7443 | continue fixing tests --- .../dsm/model/elastic/filter/Operator.java | 2 +- .../process/CollectionProcessorTest.java | 61 +++++++++---------- .../export/process/SingleProcessorTest.java | 16 ++++- 3 files changed, 45 insertions(+), 34 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java index 6d396cbf1..1dba3cfd2 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java @@ -106,6 +106,6 @@ private static boolean isNotNull(String[] splittedFilter) { splittedFilter = cleanFromEmptySpaces(splittedFilter); final int BOTTOM_SIZE_OF_IS_NOT_NULL = 4; if (splittedFilter.length < BOTTOM_SIZE_OF_IS_NOT_NULL) return false; - return Filter.IS.equals(splittedFilter[1]) && Filter.NOT.equals(splittedFilter[2]) && Filter.NULL.equals(splittedFilter[3]); + return Filter.IS.equals(splittedFilter[splittedFilter.length - 3]) && Filter.NOT.equals(splittedFilter[splittedFilter.length - 2]) && Filter.NULL.equals(splittedFilter[splittedFilter.length - 1]); } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorTest.java index 6fde1ede7..501113ffc 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/CollectionProcessorTest.java @@ -33,17 +33,21 @@ public void testProcess() throws IOException { String propertyName = "medicalRecord"; double recordId = 5; String oldValue = "mr_old"; - String json = String.format("{\"%s\":[{\"id\":%s,\"mr\":\"%s\"}]}", propertyName, recordId, oldValue); + String json = String.format("{\"%s\":[{\"medicalRcordId\":%s,\"mrProblemText\":\"%s\"}]}", propertyName, recordId, oldValue); ObjectMapper objectMapper = new ObjectMapper(); ESDsm esDsm = objectMapper.readValue(json, ESDsm.class); - NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "mr_updated"); - GeneratorPayload generatorPayload = MappingGeneratorTest.getGeneratorPayload(TestPatchUtil.MEDICAL_RECORD_COLUMN, "mr_updated", 5); + NameValue nameValue = new NameValue("m.mrProblemText", "mr_updated"); + + GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, 5); + + BaseParser valueParser = new ValueParser(); + valueParser.setPropertyInfo(new BaseGenerator.PropertyInfo(MedicalRecord.class, true)); + Processor collectionProcessor = new TestCollectionProcessor(esDsm, propertyName, generatorPayload, - instance(generatorPayload, - nameValue, new ValueParser())); + new CollectionSourceGenerator(valueParser, generatorPayload)); List> updatedList = (List>) collectionProcessor.process(); @@ -58,16 +62,21 @@ public void testProcess() throws IOException { public void updateIfExistsOrPut() throws IOException { String propertyName = "medicalRecord"; double recordId = 5; - String json = String.format("{\"%s\":[{\"id\":%s,\"mr\":\"%s\"}]}", propertyName, recordId, "value");; + String json = String.format("{\"%s\":[{\"medicalRcordId\":%s,\"mrProblemText\":\"%s\"}]}", propertyName, recordId, "value");; ObjectMapper objectMapper = new ObjectMapper(); ESDsm esDsm = objectMapper.readValue(json, ESDsm.class); - NameValue nameValue = new NameValue(TestPatchUtil.MEDICAL_RECORD_COLUMN, "val"); - GeneratorPayload generatorPayload = MappingGeneratorTest.getGeneratorPayload(TestPatchUtil.MEDICAL_RECORD_COLUMN, "val", 10); + NameValue nameValue = new NameValue("m.mrProblemText", "val"); + + GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, 10); + + BaseParser valueParser = new ValueParser(); + valueParser.setPropertyInfo(new BaseGenerator.PropertyInfo(MedicalRecord.class, true)); + CollectionProcessor collectionProcessor = new TestCollectionProcessor(esDsm, propertyName, generatorPayload, - instance(generatorPayload, nameValue, new ValueParser())); + new CollectionSourceGenerator(valueParser, generatorPayload)); List> updatedList = collectionProcessor.process(); @@ -79,43 +88,33 @@ public void updateIfExistsOrPut() throws IOException { public void updateIfExists() throws IOException { String propertyName = "medicalRecord"; double recordId = 5; - String json = String.format("{\"%s\":[{\"id\":%s,\"TEST1\":\"%s\", \"TEST2\":\"TEST_VAL2\"}]}", propertyName, recordId, "value");; + String json = String.format("{\"%s\":[{\"medicalRecordId\":%s,\"type\":\"%s\", \"mrProblemText\":\"TEST_VAL2\"}]}", propertyName, recordId, "value");; ObjectMapper objectMapper = new ObjectMapper(); ESDsm esDsm = objectMapper.readValue(json, ESDsm.class); - NameValue nameValue = new NameValue(TestPatchUtil.ADDITIONAL_VALUES_JSON, "{\"TEST1\":\"TEST_VAL\"}"); + NameValue nameValue = new NameValue("m.type", "TEST_VAL"); GeneratorPayload generatorPayload = new GeneratorPayload(nameValue, 5); - DynamicFieldsParser parser = new DynamicFieldsParser(); - parser.setDisplayType("TEXT"); - parser.setParser(new ValueParser()); - parser.setPropertyInfo(new BaseGenerator.PropertyInfo(MedicalRecord.class, true)); + + BaseParser valueParser = new ValueParser(); + valueParser.setPropertyInfo(new BaseGenerator.PropertyInfo(MedicalRecord.class, true)); + CollectionProcessor collectionProcessor = new TestCollectionProcessor(esDsm, propertyName, generatorPayload, - instance(generatorPayload, nameValue, parser)); + new CollectionSourceGenerator(valueParser, generatorPayload)); + List> updatedList = collectionProcessor.process(); updatedList.stream() - .filter(i -> i.containsKey("TEST1")) + .filter(i -> i.containsKey("type")) .findFirst() - .ifPresentOrElse(m -> m.get("TEST1").equals("TEST_VAL"), Assert::fail); + .ifPresentOrElse(m -> Assert.assertEquals("TEST_VAL", m.get("type")), Assert::fail); updatedList.stream() - .filter(i -> i.containsKey("TEST2")) + .filter(i -> i.containsKey("mrProblemText")) .findFirst() - .ifPresentOrElse(m -> m.get("TEST2").equals("TEST_VAL3"), Assert::fail); - } - - private Collector instance(GeneratorPayload generatorPayload, NameValue nameValue, BaseParser parser) { - parser.setPropertyInfo(new BaseGenerator.PropertyInfo(MappingGeneratorTest.TestPropertyClass.class, true)); - BaseGenerator sourceGenerator = new CollectionSourceGenerator(parser, generatorPayload) { - @Override - protected DBElement getDBElement() { - return TestPatchUtil.getColumnNameMap().get(nameValue.getName()); - } - }; - return sourceGenerator; + .ifPresentOrElse(m -> Assert.assertEquals("TEST_VAL2", m.get("mrProblemText")), Assert::fail); } private static class TestCollectionProcessor extends CollectionProcessor { diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/SingleProcessorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/SingleProcessorTest.java index 7b9dd426d..75db8c6f5 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/SingleProcessorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/SingleProcessorTest.java @@ -8,6 +8,7 @@ import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorFactory; import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; import org.broadinstitute.dsm.model.elastic.export.generate.SourceGeneratorFactory; +import org.broadinstitute.dsm.model.elastic.export.parse.DynamicFieldsParser; import org.broadinstitute.dsm.model.elastic.export.parse.ValueParser; import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.statics.ESObjectConstants; @@ -22,9 +23,15 @@ public class SingleProcessorTest { public void processExisting() { BaseGenerator.PropertyInfo propertyInfo = Util.TABLE_ALIAS_MAPPINGS.get(DBConstants.DDP_PARTICIPANT_ALIAS); ValueParser valueParser = new ValueParser(); + valueParser.setPropertyInfo(propertyInfo); + + DynamicFieldsParser dynamicFieldsParser = new DynamicFieldsParser(); + dynamicFieldsParser.setParser(valueParser); + dynamicFieldsParser.setDisplayType("TEXT"); + GeneratorFactory sourceGeneratorFactory = new SourceGeneratorFactory(); BaseGenerator generator = sourceGeneratorFactory.make(propertyInfo); - generator.setParser(valueParser); + generator.setParser(dynamicFieldsParser); generator.setPayload(new GeneratorPayload(new NameValue("p.additionalValuesJson", "{\"key\":\"value\"}"), 0)); ESDsm esDsm = new ESDsm(); @@ -43,9 +50,14 @@ public void processExisting() { public void processNew() { BaseGenerator.PropertyInfo propertyInfo = Util.TABLE_ALIAS_MAPPINGS.get(DBConstants.DDP_PARTICIPANT_ALIAS); ValueParser valueParser = new ValueParser(); + + DynamicFieldsParser dynamicFieldsParser = new DynamicFieldsParser(); + dynamicFieldsParser.setParser(valueParser); + dynamicFieldsParser.setDisplayType("TEXT"); + GeneratorFactory sourceGeneratorFactory = new SourceGeneratorFactory(); BaseGenerator generator = sourceGeneratorFactory.make(propertyInfo); - generator.setParser(valueParser); + generator.setParser(dynamicFieldsParser); generator.setPayload(new GeneratorPayload(new NameValue("p.additionalValuesJson", "{\"key\":\"value\"}"), 0)); ESDsm esDsm = new ESDsm(); From dc921409d58bb5c74f0ab5f16405d46ac80d406c Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 20 Jan 2022 12:30:17 +0400 Subject: [PATCH 331/405] DDP-7443 | attempt to fix IS NOT NULL problem for JSON EXTRACT --- .../dsm/model/elastic/filter/Operator.java | 10 +++++----- .../dsm/model/elastic/filter/OperatorTest.java | 8 ++------ 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java index 1dba3cfd2..e88395fd8 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java @@ -1,7 +1,6 @@ package org.broadinstitute.dsm.model.elastic.filter; import java.util.*; -import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; @@ -24,6 +23,8 @@ public enum Operator { public static final String MULTIPLE_OPTIONS_INDICATOR = "()"; public static final String UNKNOWN_OPERATOR = "Unknown operator"; + public static final List IS_NOT_NULL_LIST = Arrays.asList("IS", "NOT", + "NULL"); private String value; @@ -42,12 +43,10 @@ public static Operator extract(String filter) { String[] splittedFilter = filter.split(Filter.SPACE); if (isMultipleOptions(splittedFilter)) return MULTIPLE_OPTIONS; - else if (isNotNull(splittedFilter)) - return IS_NOT_NULL; Optional maybeOperator = Arrays.stream(splittedFilter) .filter(StringUtils::isNotBlank) .map(Operator::handleSpecialCaseOperators) - .filter(word -> Arrays.stream(Operator.values()).anyMatch(op -> op.value.equals(word))) + .filter(word -> Arrays.stream(Operator.values()).anyMatch(op -> op.value.equals(word)) || Operator.IS_NOT_NULL_LIST.contains(word)) .distinct() .reduce((prev, curr) -> String.join(Filter.SPACE, prev, curr)); if (maybeOperator.isPresent()) { @@ -64,6 +63,7 @@ else if (isNotNull(splittedFilter)) case "JSON_EXTRACT =": case "JSON_EXTRACT >=": case "JSON_EXTRACT <=": + case "JSON_EXTRACT IS NOT NULL": return JSON_EXTRACT; default: return Operator.getOperator(operator); @@ -77,7 +77,7 @@ private static String handleSpecialCaseOperators(String word) { String strOperator = StringUtils.EMPTY; for (Operator operator : Operator.values()) { int startIndex = word.indexOf(operator.value); - if (startIndex == -1) continue; + if (startIndex == -1 && !Operator.IS_NOT_NULL_LIST.contains(word)) continue; if (word.contains(Filter.OPEN_PARENTHESIS)) strOperator = word.substring(startIndex, startIndex + operator.value.length()); else strOperator = word; return strOperator; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java index 62121d27d..386f9e3ea 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java @@ -2,12 +2,8 @@ import static org.junit.Assert.*; -import java.util.Arrays; -import java.util.List; - import org.broadinstitute.dsm.model.Filter; import org.junit.Assert; -import org.junit.Ignore; import org.junit.Test; public class OperatorTest { @@ -25,7 +21,7 @@ public void getOperator() { assertEquals(Operator.EQUALS, equals); assertEquals(Operator.GREATER_THAN_EQUALS, larger); assertEquals(Operator.LESS_THAN_EQUALS, smaller); - assertEquals(Operator.IS_NOT_NULL, isNotNull); + assertEquals(Operator.IS_NOT_NULL_LIST, isNotNull); } catch (IllegalArgumentException iae) { Assert.assertTrue(true); } @@ -85,7 +81,7 @@ public void extractLessThanEqualsOperator() { public void extractIsNotNullOperator() { String filter = "m.mr_received IS NOT NULL"; Operator operator = Operator.extract(filter); - assertEquals(Operator.IS_NOT_NULL, operator); + assertEquals(Operator.IS_NOT_NULL_LIST, operator); } @Test From 98fbdb0ecff289eda84bb35262fedce283e0ce57 Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 20 Jan 2022 15:19:26 +0400 Subject: [PATCH 332/405] DDP-7443 | fix bugs in tests, make all of them pass --- .../org/broadinstitute/dsm/db/Tissue.java | 2 +- .../model/elastic/export/BaseExporter.java | 1 - .../export/ElasticDataExportAdapter.java | 3 +- .../export/ElasticMappingExportAdapter.java | 3 +- .../dsm/model/elastic/filter/Operator.java | 2 + .../model/elastic/filter/OperatorTest.java | 2 +- .../migration/BaseCollectionMigratorTest.java | 39 +++++++++++-------- .../DynamicFieldsMappingMigratorTest.java | 2 + .../migration/DynamicFieldsParserTest.java | 1 + .../migration/MedicalRecordMigratorTest.java | 20 ---------- .../OncHistoryDetailsMigratorTest.java | 19 --------- .../migration/OncHistoryMigratorTest.java | 22 ----------- .../elastic/migration/TissueMigratorTest.java | 23 ----------- 13 files changed, 34 insertions(+), 105 deletions(-) delete mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigratorTest.java delete mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigratorTest.java delete mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryMigratorTest.java delete mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigratorTest.java diff --git a/src/main/java/org/broadinstitute/dsm/db/Tissue.java b/src/main/java/org/broadinstitute/dsm/db/Tissue.java index 167c8bfc1..63f3e46fc 100644 --- a/src/main/java/org/broadinstitute/dsm/db/Tissue.java +++ b/src/main/java/org/broadinstitute/dsm/db/Tissue.java @@ -163,7 +163,7 @@ public long gethECount() { public Tissue() {} - public Tissue(long tissueId, long oncHistoryDetailId, String notes, Integer countReceived, String tissueType, + public Tissue(long tissueId, long oncHistoryDetailId, String notes, long countReceived, String tissueType, String tissueSite, String tumorType, String hE, String pathologyReport, String collaboratorSampleId, String blockSent, String scrollsReceived, String skId, String smId, String sentGp, String firstSmId, String additionalValuesJson, String expectedReturn, String returnDate, diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java index 8d19c62eb..53e21d00a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/BaseExporter.java @@ -6,7 +6,6 @@ import org.elasticsearch.client.RestHighLevelClient; public abstract class BaseExporter implements Exportable, ExportableHelper { - protected RestHighLevelClient clientInstance = ElasticSearchUtil.getClientInstance(); protected RequestPayload requestPayload; protected Map source; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticDataExportAdapter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticDataExportAdapter.java index 043009b17..69802d643 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticDataExportAdapter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticDataExportAdapter.java @@ -3,6 +3,7 @@ import java.io.IOException; import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.RequestOptions; import org.slf4j.Logger; @@ -19,7 +20,7 @@ public void export() { .doc(source) .retryOnConflict(5); try { - clientInstance.update(updateRequest, RequestOptions.DEFAULT); + ElasticSearchUtil.getClientInstance().update(updateRequest, RequestOptions.DEFAULT); } catch (IOException e) { throw new RuntimeException("Error occurred while exporting data to ES", e); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticMappingExportAdapter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticMappingExportAdapter.java index 88bab66b0..a72fb5521 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticMappingExportAdapter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticMappingExportAdapter.java @@ -2,6 +2,7 @@ import java.io.IOException; +import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.indices.PutMappingRequest; import org.slf4j.Logger; @@ -17,7 +18,7 @@ public void export() { PutMappingRequest putMappingRequest = new PutMappingRequest(requestPayload.getIndex()); putMappingRequest.source(source); try { - clientInstance.indices().putMapping(putMappingRequest, RequestOptions.DEFAULT); + ElasticSearchUtil.getClientInstance().indices().putMapping(putMappingRequest, RequestOptions.DEFAULT); } catch (IOException e) { throw new RuntimeException("Error occurred while updating mapping to ES", e); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java index e88395fd8..fb1b6705d 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java @@ -60,6 +60,8 @@ public static Operator extract(String filter) { return Operator.DATE_GREATER; case "DATE =": return Operator.DATE; + case "NOT <=>": + return Operator.DIAMOND_EQUALS; case "JSON_EXTRACT =": case "JSON_EXTRACT >=": case "JSON_EXTRACT <=": diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java index 386f9e3ea..922fa6102 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java @@ -81,7 +81,7 @@ public void extractLessThanEqualsOperator() { public void extractIsNotNullOperator() { String filter = "m.mr_received IS NOT NULL"; Operator operator = Operator.extract(filter); - assertEquals(Operator.IS_NOT_NULL_LIST, operator); + assertEquals(Operator.IS_NOT_NULL, operator); } @Test diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigratorTest.java index 2fe97beb8..0c7959523 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/BaseCollectionMigratorTest.java @@ -7,8 +7,7 @@ import org.broadinstitute.dsm.db.OncHistoryDetail; import org.broadinstitute.dsm.db.Tissue; -import org.broadinstitute.dsm.model.elastic.export.generate.GeneratorPayload; -import org.broadinstitute.dsm.model.elastic.export.parse.Parser; +import org.elasticsearch.client.RestHighLevelClient; import org.junit.Assert; import org.junit.Test; @@ -17,34 +16,42 @@ public class BaseCollectionMigratorTest { @Test public void transformObject() { BaseCollectionMigrator baseCollectionMigrator = new MockBaseCollectionMigrator("index", "realm", "object"); - baseCollectionMigrator.transformObject(mockData()); + baseCollectionMigrator.transformObject(mockOncHistoryDetail()); Map objectMap = baseCollectionMigrator.transformedList.get(0); - Object primaryId = objectMap.get("id"); - Assert.assertEquals("23", primaryId); - Map tissue = (Map) ((List) objectMap.get("tissues")).get(0); - Object tissuePrimaryId = tissue.get("id"); - Assert.assertEquals("11", tissuePrimaryId); - Assert.assertEquals(tissuePrimaryId, tissue.get("tissueId")); + Object primaryId = objectMap.get("oncHistoryDetailId"); + Assert.assertEquals(23L, primaryId); + + baseCollectionMigrator.transformObject(mockTissues()); + Map stringObjectMap = baseCollectionMigrator.transformedList.get(0); + Assert.assertEquals(11L, stringObjectMap.get("tissueId")); + Assert.assertEquals("notes", stringObjectMap.get("notes")); + } + + private List mockOncHistoryDetail() { + OncHistoryDetail oncHistoryDetail = + new OncHistoryDetail(23, 0, null, null, null, null, null, null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null, null, mockTissues(), null, null, false); + return Collections.singletonList(oncHistoryDetail); } - private List mockData() { + private List mockTissues() { List fieldValue = new ArrayList<>(List.of(new Tissue(11, 22, - null, null, null, "awdwadawdawdawd", null, null, null, null, null, null, + "notes", 0, null, "awdwadawdawdawd", null, null, null, null, null, null, null, null, "Awdawd", null, null, null, null, null, null, null, null, 0, 0, 0, 0), new Tissue(555, 777, - null, null, null, null, null, null, null, "awdawd", null, null, + null, 0, null, null, null, null, null, "awdawd", null, null, null, null, "awdawddwa", null, null, null, null, null, null, null, null, 0, 0, 0, 0))); - OncHistoryDetail oncHistoryDetail = - new OncHistoryDetail(23, 0, null, null, null, null, null, null, null, null, null, null, null, null, null, - null, null, null, null, null, null, null, null, null, fieldValue, null, null, false); - return Collections.singletonList(oncHistoryDetail); + return fieldValue; } static class MockBaseCollectionMigrator extends BaseCollectionMigrator { + protected RestHighLevelClient clientInstance; + public MockBaseCollectionMigrator(String index, String realm, String object) { super(index, realm, object); + } @Override diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java index 7c173deff..df4c7c8bf 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java @@ -3,6 +3,7 @@ import org.broadinstitute.dsm.TestHelper; import org.broadinstitute.dsm.model.elastic.export.Exportable; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Test; import java.util.Arrays; @@ -16,6 +17,7 @@ public static void setUp() { } @Test + @Ignore public void testExport() { final String index = "participants_structured.cmi.angio"; final String study = "angio"; diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsParserTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsParserTest.java index 39bc6572f..1005fab74 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsParserTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsParserTest.java @@ -18,6 +18,7 @@ public void parse() { DynamicFieldsParser dynamicFieldsParser = new DynamicFieldsParser(); dynamicFieldsParser.setDisplayType(displayType); dynamicFieldsParser.setPossibleValuesJson(possibleValuesJson); + dynamicFieldsParser.setParser(new TypeParser()); Map mapping = (Map) dynamicFieldsParser.parse(displayType); Object date = mapping.get(MappingGenerator.TYPE); assertEquals(TypeParser.DATE, date); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigratorTest.java deleted file mode 100644 index 2971c7600..000000000 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/MedicalRecordMigratorTest.java +++ /dev/null @@ -1,20 +0,0 @@ -package org.broadinstitute.dsm.model.elastic.migration; - -import org.broadinstitute.dsm.TestHelper; -import org.junit.BeforeClass; -import org.junit.Test; - -public class MedicalRecordMigratorTest { - - @BeforeClass - public static void setUp() { - TestHelper.setupDB(); - } - - @Test - public void export() { - MedicalRecordMigrator medicalRecordMigrator = new MedicalRecordMigrator("participants_structured.cmi.angio", "angio"); - medicalRecordMigrator.export(); - } - -} \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigratorTest.java deleted file mode 100644 index d79f5a5bd..000000000 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryDetailsMigratorTest.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.broadinstitute.dsm.model.elastic.migration; - -import org.broadinstitute.dsm.TestHelper; -import org.junit.BeforeClass; -import org.junit.Test; - -public class OncHistoryDetailsMigratorTest { - - @BeforeClass - public static void setUp() { - TestHelper.setupDB(); - } - - @Test - public void export() { - OncHistoryDetailsMigrator oncHistoryDetailsMigrator = new OncHistoryDetailsMigrator("participants_structured.cmi.angio", "angio"); - oncHistoryDetailsMigrator.export(); - } -} \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryMigratorTest.java deleted file mode 100644 index 74e8b96e3..000000000 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/OncHistoryMigratorTest.java +++ /dev/null @@ -1,22 +0,0 @@ -package org.broadinstitute.dsm.model.elastic.migration; - -import static org.junit.Assert.*; - -import org.broadinstitute.dsm.TestHelper; -import org.junit.BeforeClass; -import org.junit.Test; - -public class OncHistoryMigratorTest { - - @BeforeClass - public static void setUp() { - TestHelper.setupDB(); - } - - @Test - public void export() { - OncHistoryMigrator oncHistoryMigrator = new OncHistoryMigrator("participants_structured.cmi.angio", "angio"); - oncHistoryMigrator.export(); - } - -} \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigratorTest.java deleted file mode 100644 index 0aef6292d..000000000 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/TissueMigratorTest.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.broadinstitute.dsm.model.elastic.migration; - -import static org.junit.Assert.*; - -import java.util.Map; - -import org.broadinstitute.dsm.TestHelper; -import org.junit.BeforeClass; -import org.junit.Test; - -public class TissueMigratorTest { - - @BeforeClass - public static void setUp() { - TestHelper.setupDB(); - } - - @Test - public void getDataByRealm() { - TissueMigrator tissueMigrator = new TissueMigrator("participants_structured.cmi.angio", "angio"); - tissueMigrator.export(); - } -} \ No newline at end of file From 0d9c533be3f8c855f46c30105f796926229a16ca Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 20 Jan 2022 16:30:53 +0400 Subject: [PATCH 333/405] DDP-7443 | pass ddpInstanceId in KitRequestShipping.writeReuqest() in order to get realm and ES index, start creating necessary UpsertPatch dependencies --- .../dsm/db/KitRequestShipping.java | 28 +++++++++++++------ .../dsm/route/CreateBSPDummyKitRoute.java | 2 +- .../route/CreateClinicalDummyKitRoute.java | 2 +- .../dsm/route/KitUploadRoute.java | 4 +-- .../dsm/util/DDPKitRequest.java | 8 +++--- 5 files changed, 28 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java index 5da419abf..7daabb6ad 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java @@ -18,8 +18,11 @@ import org.broadinstitute.dsm.model.*; import org.broadinstitute.dsm.model.ddp.DDPParticipant; import org.broadinstitute.dsm.model.ddp.KitDetail; +import org.broadinstitute.dsm.model.elastic.export.RequestPayload; import org.broadinstitute.dsm.model.elastic.export.generate.Generator; +import org.broadinstitute.dsm.model.elastic.export.painless.NestedScriptBuilder; import org.broadinstitute.dsm.model.elastic.export.painless.ParamsGenerator; +import org.broadinstitute.dsm.model.elastic.export.painless.ScriptBuilder; import org.broadinstitute.dsm.model.elastic.export.painless.UpsertPainless; import org.broadinstitute.dsm.statics.ApplicationConfigConstants; import org.broadinstitute.dsm.statics.DBConstants; @@ -796,15 +799,15 @@ public static KitRequestShipping getKitRequest(@NonNull String kitRequestId) { //adding kit request to db (called by hourly job to add kits into DSM) public static void addKitRequests(@NonNull String instanceId, @NonNull KitDetail kitDetail, @NonNull int kitTypeId, - @NonNull KitRequestSettings kitRequestSettings, String collaboratorParticipantId, String externalOrderNumber, String uploadReason) { + @NonNull KitRequestSettings kitRequestSettings, String collaboratorParticipantId, String externalOrderNumber, String uploadReason, DDPInstance ddpInstance) { addKitRequests(instanceId, kitDetail.getKitType(), kitDetail.getParticipantId(), kitDetail.getKitRequestId(), kitTypeId, kitRequestSettings, - collaboratorParticipantId, kitDetail.isNeedsApproval(), externalOrderNumber, uploadReason); + collaboratorParticipantId, kitDetail.isNeedsApproval(), externalOrderNumber, uploadReason, ddpInstance); } //adding kit request to db (called by hourly job to add kits into DSM) public static void addKitRequests(@NonNull String instanceId, @NonNull String kitType, @NonNull String participantId, @NonNull String kitRequestId, @NonNull int kitTypeId, @NonNull KitRequestSettings kitRequestSettings, String collaboratorParticipantId, boolean needsApproval, String externalOrderNumber, - String uploadReason) { + String uploadReason, DDPInstance ddpInstance) { inTransaction((conn) -> { String errorMessage = ""; String collaboratorSampleId = null; @@ -822,7 +825,7 @@ public static void addKitRequests(@NonNull String instanceId, @NonNull String ki } } writeRequest(instanceId, kitRequestId, kitTypeId, participantId, collaboratorParticipantId, collaboratorSampleId, - "SYSTEM", null, errorMessage, externalOrderNumber, needsApproval, uploadReason); + "SYSTEM", null, errorMessage, externalOrderNumber, needsApproval, uploadReason, ddpInstance); return null; }); } @@ -831,9 +834,9 @@ public static void addKitRequests(@NonNull String instanceId, @NonNull String ki // called by // 1. hourly job to add kit requests into db // 2. kit upload - public static String writeRequest(@NonNull String instanceId, @NonNull String ddpKitRequestId, @NonNull int kitTypeId, + public static String writeRequest(@NonNull String instanceId, @NonNull String ddpKitRequestId, int kitTypeId, @NonNull String ddpParticipantId, String collaboratorPatientId, String collaboratorSampleId, - @NonNull String createdBy, String addressIdTo, String errorMessage, String externalOrderNumber, boolean needsApproval, String uploadReason) { + @NonNull String createdBy, String addressIdTo, String errorMessage, String externalOrderNumber, boolean needsApproval, String uploadReason, DDPInstance ddpInstance) { SimpleResult results = inTransaction((conn) -> { SimpleResult dbVals = new SimpleResult(0); try (PreparedStatement insertKitRequest = conn.prepareStatement(TransactionWrapper.getSqlFromConfig(ApplicationConfigConstants.INSERT_KIT_REQUEST), Statement.RETURN_GENERATED_KEYS)) { @@ -871,8 +874,17 @@ public static String writeRequest(@NonNull String instanceId, @NonNull String dd throw new RuntimeException("Error adding kit request w/ ddpKitRequestId " + ddpKitRequestId, results.resultException); } -// new ParamsGenerator("", ) -// new UpsertPainless() + + KitRequestShipping kitRequestShipping = new KitRequestShipping(ddpParticipantId, collaboratorPatientId, collaboratorSampleId, null, + null, null, 0, 0, null, null, null, null, null, null, 0, false, errorMessage, 0, null, + 0, null, null, false, null, 0, null, externalOrderNumber, false, null, createdBy, null, + null, null, 0, false, uploadReason, null, null, null); + + Generator paramsGenerator = new ParamsGenerator(kitRequestShipping, ddpInstance.getName()); + RequestPayload requestPayload = new RequestPayload(ddpInstance.getParticipantIndexES(), ddpParticipantId); + ScriptBuilder scriptBuilder = new NestedScriptBuilder("kitRequestShipping", "ddpKitRequestId"); + UpsertPainless upsertPainless = new UpsertPainless(paramsGenerator, requestPayload, scriptBuilder); + upsertPainless.export(); logger.info("Added kitRequest w/ ddpKitRequestId " + ddpKitRequestId); return (String) results.resultValue; diff --git a/src/main/java/org/broadinstitute/dsm/route/CreateBSPDummyKitRoute.java b/src/main/java/org/broadinstitute/dsm/route/CreateBSPDummyKitRoute.java index 46207af59..f26a52240 100644 --- a/src/main/java/org/broadinstitute/dsm/route/CreateBSPDummyKitRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/CreateBSPDummyKitRoute.java @@ -44,7 +44,7 @@ public Object handle(Request request, Response response) throws Exception { //if instance not null String dsmKitRequestId = KitRequestShipping.writeRequest(mockDdpInstance.getDdpInstanceId(), mercuryKitRequestId, kitTypeId, ddpParticipantId, participantCollaboratorId, collaboratorSampleId, - USER_ID, "", "", "", false, ""); + USER_ID, "", "", "", false, "", null); new BSPDummyKitDao().updateKitLabel(kitLabel, dsmKitRequestId); } logger.info("Returning 200 to Mercury"); diff --git a/src/main/java/org/broadinstitute/dsm/route/CreateClinicalDummyKitRoute.java b/src/main/java/org/broadinstitute/dsm/route/CreateClinicalDummyKitRoute.java index 8eda32f2e..3c2a40a8d 100644 --- a/src/main/java/org/broadinstitute/dsm/route/CreateClinicalDummyKitRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/CreateClinicalDummyKitRoute.java @@ -57,7 +57,7 @@ public Object handle(Request request, Response response) { //if instance not null String dsmKitRequestId = KitRequestShipping.writeRequest(ddpInstance.getDdpInstanceId(), kitRequestId, desiredKitType.getKitId(), ddpParticipantId, participantCollaboratorId, collaboratorSampleId, - USER_ID, "", "", "", false, ""); + USER_ID, "", "", "", false, "", ddpInstance); new BSPDummyKitDao().updateKitLabel(kitLabel, dsmKitRequestId); }, () -> { diff --git a/src/main/java/org/broadinstitute/dsm/route/KitUploadRoute.java b/src/main/java/org/broadinstitute/dsm/route/KitUploadRoute.java index 303d78257..808e8b163 100644 --- a/src/main/java/org/broadinstitute/dsm/route/KitUploadRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/KitUploadRoute.java @@ -369,7 +369,7 @@ private void addKitRequest(Connection conn, String kitTypeName, KitRequestSettin KitRequestShipping.writeRequest(ddpInstance.getDdpInstanceId(), shippingId, kitTypeId, kit.getParticipantId().trim(), collaboratorParticipantId, collaboratorSampleId, userId, addressId, - errorMessage, externalOrderNumber, false, uploadReason); + errorMessage, externalOrderNumber, false, uploadReason, ddpInstance); kit.setShippingId(shippingId); kit.setExternalOrderNumber(externalOrderNumber); } @@ -390,7 +390,7 @@ private void addKitRequest(Connection conn, String kitTypeName, KitRequestSettin KitRequestShipping.writeRequest(ddpInstance.getDdpInstanceId(), shippingId, kitTypeId, kit.getParticipantId().trim(), collaboratorParticipantId, collaboratorSampleId, userId, addressId, - errorMessage, kit.getExternalOrderNumber(), false, uploadReason); + errorMessage, kit.getExternalOrderNumber(), false, uploadReason, ddpInstance); kit.setShippingId(shippingId); } } diff --git a/src/main/java/org/broadinstitute/dsm/util/DDPKitRequest.java b/src/main/java/org/broadinstitute/dsm/util/DDPKitRequest.java index ae3ba7120..1191d86c6 100644 --- a/src/main/java/org/broadinstitute/dsm/util/DDPKitRequest.java +++ b/src/main/java/org/broadinstitute/dsm/util/DDPKitRequest.java @@ -74,10 +74,10 @@ public void requestAndWriteKitRequests(List latestKitRequests) if (kitHasSubKits) { List subKits = kitRequestSettings.getSubKits(); - addSubKits(subKits, kitDetail, collaboratorParticipantId, kitRequestSettings, latestKit.getInstanceID(), null); + addSubKits(subKits, kitDetail, collaboratorParticipantId, kitRequestSettings, latestKit.getInstanceID(), null, ddpInstance); } else { KitRequestShipping.addKitRequests(latestKit.getInstanceID(), kitDetail, kitType.getKitTypeId(), - kitRequestSettings, collaboratorParticipantId, null, null); + kitRequestSettings, collaboratorParticipantId, null, null, ddpInstance); } } else { logger.error("ES profile data was empty for participant with ddp_kit_request_id " + kitDetail.getKitRequestId()); @@ -120,7 +120,7 @@ public void requestAndWriteKitRequests(List latestKitRequests) } private String addSubKits(@NonNull List subKits, @NonNull KitDetail kitDetail, @NonNull String collaboratorParticipantId, - @NonNull KitRequestSettings kitRequestSettings, @NonNull String instanceId, String uploadReason) { + @NonNull KitRequestSettings kitRequestSettings, @NonNull String instanceId, String uploadReason, DDPInstance ddpInstance) { int subCounter = 0; String externalOrderNumber = null; if (StringUtils.isNotBlank(kitRequestSettings.getExternalShipper())) { @@ -131,7 +131,7 @@ private String addSubKits(@NonNull List subKits, @NonNull KitDetail //kitRequestId needs to stay unique -> add `_[SUB_COUNTER]` to it KitRequestShipping.addKitRequests(instanceId, subKit.getKitName(), kitDetail.getParticipantId(), subCounter == 0 ? kitDetail.getKitRequestId() : kitDetail.getKitRequestId() + "_" + subCounter, subKit.getKitTypeId(), kitRequestSettings, - collaboratorParticipantId, kitDetail.isNeedsApproval(), externalOrderNumber, uploadReason); + collaboratorParticipantId, kitDetail.isNeedsApproval(), externalOrderNumber, uploadReason, ddpInstance); subCounter = subCounter + 1; } } From 5738f57221f3aeb294ac113ef3a9647b0e909423 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 20 Jan 2022 17:05:10 +0400 Subject: [PATCH 334/405] DDP-7443 | add upsert painless to deactivate kit request --- .../dsm/db/KitRequestShipping.java | 40 +++++++++++++------ .../export/painless/UpsertPainless.java | 1 + .../dsm/route/KitDeactivationRoute.java | 6 ++- .../dsm/route/KitExpressRoute.java | 2 +- .../org/broadinstitute/dsm/util/KitUtil.java | 5 ++- 5 files changed, 38 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java index 7daabb6ad..c3ee3b221 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java @@ -10,6 +10,7 @@ import org.broadinstitute.ddp.db.SimpleResult; import org.broadinstitute.ddp.db.TransactionWrapper; import org.broadinstitute.dsm.DSMServer; +import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; import org.broadinstitute.dsm.db.structure.ColumnName; import org.broadinstitute.dsm.db.structure.DbDateConversion; import org.broadinstitute.dsm.db.structure.SqlDateConverter; @@ -736,8 +737,9 @@ private static void getAddressPerEasypost(DDPInstance ddpInstance, KitRequestShi } } - public static void deactivateKitRequest(@NonNull long kitRequestId, @NonNull String reason, String easypostApiKey, - @NonNull String userId) { + public static void deactivateKitRequest(long kitRequestId, @NonNull String reason, String easypostApiKey, + @NonNull String userId, + DDPInstanceDto ddpInstanceDto) { SimpleResult results = inTransaction((conn) -> { SimpleResult dbVals = new SimpleResult(); try (PreparedStatement stmt = conn.prepareStatement(UPDATE_KIT_DEACTIVATION)) { @@ -762,6 +764,19 @@ public static void deactivateKitRequest(@NonNull long kitRequestId, @NonNull Str if (results.resultException != null) { throw new RuntimeException("Error setting kitRequest to deactivated w/ dsm_kit_request_id " + kitRequestId, results.resultException); } + if (Objects.nonNull(ddpInstanceDto)) { + new KitRequestShipping() + KitRequestShipping kitRequestShipping = new KitRequestShipping(ddpParticipantId, collaboratorPatientId, collaboratorSampleId, null, + null, null, Long.parseLong(ddpKitRequestId), 0, null, null, null, null, null, null, 0, false, errorMessage, 0, null, + 0, null, null, false, null, 0, null, externalOrderNumber, false, null, createdBy, null, + null, null, 0, false, uploadReason, null, null, null); + + Generator paramsGenerator = new ParamsGenerator(kitRequestShipping, ddpInstanceDto.getInstanceName()); + RequestPayload requestPayload = new RequestPayload(ddpInstanceDto.getEsParticipantIndex(), ddpParticipantId); + ScriptBuilder scriptBuilder = new NestedScriptBuilder("kitRequestShipping", "ddpKitRequestId"); + UpsertPainless upsertPainless = new UpsertPainless(paramsGenerator, requestPayload, scriptBuilder); + upsertPainless.export(); + } else { if (easypostApiKey != null) { KitRequestShipping.refundKit(kitRequestId, easypostApiKey); @@ -874,17 +889,18 @@ public static String writeRequest(@NonNull String instanceId, @NonNull String dd throw new RuntimeException("Error adding kit request w/ ddpKitRequestId " + ddpKitRequestId, results.resultException); } + if (Objects.nonNull(ddpInstance)) { + KitRequestShipping kitRequestShipping = new KitRequestShipping(ddpParticipantId, collaboratorPatientId, collaboratorSampleId, null, + null, null, Long.parseLong(ddpKitRequestId), 0, null, null, null, null, null, null, 0, false, errorMessage, 0, null, + 0, null, null, false, null, 0, null, externalOrderNumber, false, null, createdBy, null, + null, null, 0, false, uploadReason, null, null, null); - KitRequestShipping kitRequestShipping = new KitRequestShipping(ddpParticipantId, collaboratorPatientId, collaboratorSampleId, null, - null, null, 0, 0, null, null, null, null, null, null, 0, false, errorMessage, 0, null, - 0, null, null, false, null, 0, null, externalOrderNumber, false, null, createdBy, null, - null, null, 0, false, uploadReason, null, null, null); - - Generator paramsGenerator = new ParamsGenerator(kitRequestShipping, ddpInstance.getName()); - RequestPayload requestPayload = new RequestPayload(ddpInstance.getParticipantIndexES(), ddpParticipantId); - ScriptBuilder scriptBuilder = new NestedScriptBuilder("kitRequestShipping", "ddpKitRequestId"); - UpsertPainless upsertPainless = new UpsertPainless(paramsGenerator, requestPayload, scriptBuilder); - upsertPainless.export(); + Generator paramsGenerator = new ParamsGenerator(kitRequestShipping, ddpInstance.getName()); + RequestPayload requestPayload = new RequestPayload(ddpInstance.getParticipantIndexES(), ddpParticipantId); + ScriptBuilder scriptBuilder = new NestedScriptBuilder("kitRequestShipping", "ddpKitRequestId"); + UpsertPainless upsertPainless = new UpsertPainless(paramsGenerator, requestPayload, scriptBuilder); + upsertPainless.export(); + } logger.info("Added kitRequest w/ ddpKitRequestId " + ddpKitRequestId); return (String) results.resultValue; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/UpsertPainless.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/UpsertPainless.java index 04fff39d0..06777a226 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/UpsertPainless.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/UpsertPainless.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.index.reindex.UpdateByQueryRequest; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; diff --git a/src/main/java/org/broadinstitute/dsm/route/KitDeactivationRoute.java b/src/main/java/org/broadinstitute/dsm/route/KitDeactivationRoute.java index b3092203c..3f5f31736 100644 --- a/src/main/java/org/broadinstitute/dsm/route/KitDeactivationRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/KitDeactivationRoute.java @@ -9,6 +9,8 @@ import org.broadinstitute.dsm.db.DDPInstance; import org.broadinstitute.dsm.db.InstanceSettings; import org.broadinstitute.dsm.db.KitRequestShipping; +import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; +import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; import org.broadinstitute.dsm.db.dto.settings.InstanceSettingsDto; import org.broadinstitute.dsm.model.Value; import org.broadinstitute.dsm.security.RequestHandler; @@ -25,6 +27,7 @@ import spark.Response; import java.util.Map; +import java.util.Optional; public class KitDeactivationRoute extends RequestHandler { @@ -44,12 +47,13 @@ public Object processRequest(Request request, Response response, String userId) boolean deactivate = request.url().toLowerCase().contains("deactivate"); KitRequestShipping kitRequest = KitRequestShipping.getKitRequest(kitRequestId); String realm = kitRequest.getRealm(); + Optional ddpInstanceByInstanceName = new DDPInstanceDao().getDDPInstanceByInstanceName(realm); if ( UserUtil.checkUserAccess(realm, userId, "kit_deactivation", userIdRequest)) { if (deactivate) { JsonObject jsonObject = new JsonParser().parse(request.body()).getAsJsonObject(); String reason = jsonObject.get("reason").getAsString(); KitRequestShipping.deactivateKitRequest(Long.parseLong(kitRequestId), reason, - DSMServer.getDDPEasypostApiKey(realm), userIdRequest); + DSMServer.getDDPEasypostApiKey(realm), userIdRequest, ddpInstanceByInstanceName.orElse(null)); } else { QueryParamsMap queryParams = request.queryMap(); diff --git a/src/main/java/org/broadinstitute/dsm/route/KitExpressRoute.java b/src/main/java/org/broadinstitute/dsm/route/KitExpressRoute.java index ccb8dae17..a031a6563 100644 --- a/src/main/java/org/broadinstitute/dsm/route/KitExpressRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/KitExpressRoute.java @@ -72,7 +72,7 @@ public void expressKitRequest(@NonNull String kitRequestId, @NonNull String user KitRequestShipping kitRequest = KitRequestShipping.getKitRequest(kitRequestId); //deactivate kit which is already in db and refund the label KitRequestShipping.deactivateKitRequest(Long.valueOf(kitRequestId), KitRequestShipping.DEACTIVATION_REASON, - DSMServer.getDDPEasypostApiKey(kitRequest.getRealm()), userId); + DSMServer.getDDPEasypostApiKey(kitRequest.getRealm()), userId, ddpInstanceByInstanceName.orElse(null)); //add new kit into db KitRequestShipping.reactivateKitRequest(kitRequestId); diff --git a/src/main/java/org/broadinstitute/dsm/util/KitUtil.java b/src/main/java/org/broadinstitute/dsm/util/KitUtil.java index 50faf95da..01a4502e3 100644 --- a/src/main/java/org/broadinstitute/dsm/util/KitUtil.java +++ b/src/main/java/org/broadinstitute/dsm/util/KitUtil.java @@ -143,7 +143,7 @@ public static void createLabel(List kitsLabelTriggered) { else { KitRequestShipping.deactivateKitRequest(Long.parseLong(kitLabelTriggered.getDsmKitRequestId()), "Participant not found", null, - "System"); + "System", ddpInstanceByInstanceName.orElse(null)); logger.error("Didn't find participant " + kitLabelTriggered.getDdpParticipantId()); } } @@ -525,7 +525,8 @@ public static List findSpecialBehaviorKits(@NonNull Notifica boolean specialBehavior = InstanceSettings.shouldKitBehaveDifferently(participant, uploaded); if (specialBehavior) { KitRequestShipping.deactivateKitRequest(kit.getDsmKitRequestId(), SYSTEM_AUTOMATICALLY_DEACTIVATED + ": " + uploaded.getValue(), - DSMServer.getDDPEasypostApiKey(ddpInstance.getName()), "System"); + DSMServer.getDDPEasypostApiKey(ddpInstance.getName()), "System", + ddpInstanceByInstanceName.orElse(null)); if (InstanceSettings.TYPE_NOTIFICATION.equals(uploaded.getType())) { String message = kitType.getName() + " kit for participant " + kit.getParticipantId() + " (" + kit.getCollaboratorParticipantId() + ") was deactivated per background job
. " + uploaded.getValue(); From c802517ac0b9220f6a163d2caa0635b2cb77179f Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 20 Jan 2022 19:47:41 +0400 Subject: [PATCH 335/405] DDP-7121 | add more operator combination --- src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java | 4 +++- .../org/broadinstitute/dsm/model/elastic/filter/Operator.java | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java index 0ad4d8169..1d4ad596a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java @@ -40,7 +40,7 @@ public class ESDsm { @SerializedName(ESObjectConstants.DIAGNOSIS_YEAR) Object diagnosisYear; - @SerializedName(ESObjectConstants.TISSUE_RECORDS) + @SerializedName(ESObjectConstants.TISSUE) List tissue; @SerializedName(ESObjectConstants.MEDICAL_RECORD) @@ -61,6 +61,8 @@ public class ESDsm { List> medicalRecords; + List> tissueRecords; + public List getTissue() { if (tissue == null) tissue = Collections.emptyList(); return tissue; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java index 6d396cbf1..d8fa4d926 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java @@ -61,9 +61,13 @@ else if (isNotNull(splittedFilter)) return Operator.DATE_GREATER; case "DATE =": return Operator.DATE; + case "NOT <=>": + return Operator.DIAMOND_EQUALS; case "JSON_EXTRACT =": case "JSON_EXTRACT >=": case "JSON_EXTRACT <=": + case "JSON_EXTRACT LIKE": + case "JSON_EXTRACT IS NOT NULL": return JSON_EXTRACT; default: return Operator.getOperator(operator); From abad446713634d06c0e34b53b466a4022134d7c7 Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 24 Jan 2022 11:20:48 +0400 Subject: [PATCH 336/405] DDP-7443 | add ES export to deactivateKitRequest --- .../dsm/db/KitRequestShipping.java | 104 ++++++++++-------- .../export/painless/UpsertPainless.java | 22 ++-- .../dsm/route/KitExpressRoute.java | 1 + .../dsm/route/ParticipantExitRoute.java | 6 +- 4 files changed, 76 insertions(+), 57 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java index c3ee3b221..1034ed207 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java @@ -7,6 +7,7 @@ import lombok.Data; import lombok.NonNull; import org.apache.commons.lang3.StringUtils; +import org.apache.lucene.search.join.ScoreMode; import org.broadinstitute.ddp.db.SimpleResult; import org.broadinstitute.ddp.db.TransactionWrapper; import org.broadinstitute.dsm.DSMServer; @@ -19,7 +20,6 @@ import org.broadinstitute.dsm.model.*; import org.broadinstitute.dsm.model.ddp.DDPParticipant; import org.broadinstitute.dsm.model.ddp.KitDetail; -import org.broadinstitute.dsm.model.elastic.export.RequestPayload; import org.broadinstitute.dsm.model.elastic.export.generate.Generator; import org.broadinstitute.dsm.model.elastic.export.painless.NestedScriptBuilder; import org.broadinstitute.dsm.model.elastic.export.painless.ParamsGenerator; @@ -33,6 +33,8 @@ import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.broadinstitute.dsm.util.KitUtil; import org.eclipse.jetty.util.StringUtil; +import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.index.query.NestedQueryBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -152,7 +154,7 @@ public class KitRequestShipping extends KitRequest { private static final String SEARCH_MF_BAR = "MF_BAR"; @ColumnName(DBConstants.DSM_KIT_ID) - private long dsmKitId; + private Long dsmKitId; @ColumnName(DBConstants.LABEL_URL_TO) private String labelUrlTo; @@ -191,28 +193,28 @@ public class KitRequestShipping extends KitRequest { @ColumnName (DBConstants.DSM_SCAN_DATE) @DbDateConversion(SqlDateConverter.EPOCH) - private long scanDate; + private Long scanDate; @ColumnName (DBConstants.DSM_RECEIVE_DATE) @DbDateConversion(SqlDateConverter.EPOCH) - private long receiveDate; + private Long receiveDate; @ColumnName (DBConstants.DSM_DEACTIVATED_DATE) @DbDateConversion(SqlDateConverter.EPOCH) - private long deactivatedDate; + private Long deactivatedDate; @ColumnName (DBConstants.EXPRESS) - private boolean express; + private Boolean express; @ColumnName (DBConstants.EASYPOST_TO_ID) private String easypostToId; - private long labelTriggeredDate; + private Long labelTriggeredDate; @ColumnName (DBConstants.NO_RETURN) - private boolean noReturn; + private Boolean noReturn; @ColumnName (DBConstants.ERROR) - private boolean error; + private Boolean error; @ColumnName(DBConstants.MESSAGE) private String message; @@ -236,16 +238,16 @@ public class KitRequestShipping extends KitRequest { private String upsReturnStatus; @ColumnName (DBConstants.CARE_EVOLVE) - private boolean careEvolve; + private Boolean careEvolve; @ColumnName (DBConstants.UPLOAD_REASON) private String uploadReason; public KitRequestShipping() {} - public KitRequestShipping(String collaboratorParticipantId, String kitTypeName, long dsmKitRequestId, long scanDate, boolean error, - long receiveDate, long deactivatedDate, String testResult, - String upsTrackingStatus, String upsReturnStatus, String externalOrderStatus, String externalOrderNumber, long externalOrderDate, boolean careEvolve, String uploadReason) { + public KitRequestShipping(String collaboratorParticipantId, String kitTypeName, Long dsmKitRequestId, Long scanDate, Boolean error, + Long receiveDate, Long deactivatedDate, String testResult, + String upsTrackingStatus, String upsReturnStatus, String externalOrderStatus, String externalOrderNumber, Long externalOrderDate, Boolean careEvolve, String uploadReason) { this(null, collaboratorParticipantId, null, null, null, kitTypeName, dsmKitRequestId, 0, null, null, null, null, null, null, scanDate, error, null, receiveDate, null, deactivatedDate, null, null, false, null, 0, null, externalOrderNumber, false, externalOrderStatus, null, testResult, @@ -259,7 +261,7 @@ public KitRequestShipping(String participantId, String collaboratorParticipantId receiveDateString, hruid, gender); } - public KitRequestShipping(long dsmKitRequestId, long dsmKitId, String easypostToId, String easypostAddressId, boolean error, + public KitRequestShipping(Long dsmKitRequestId, Long dsmKitId, String easypostToId, String easypostAddressId, Boolean error, String message) { this(null, null, null, null, null, null, dsmKitRequestId, dsmKitId, null, null, null, null, null, null, 0, error, message, 0, @@ -737,22 +739,23 @@ private static void getAddressPerEasypost(DDPInstance ddpInstance, KitRequestShi } } - public static void deactivateKitRequest(long kitRequestId, @NonNull String reason, String easypostApiKey, + public static void deactivateKitRequest(long dsmKitRequestId, @NonNull String deactivationReason, String easypostApiKey, @NonNull String userId, DDPInstanceDto ddpInstanceDto) { + long deactivatedDate = System.currentTimeMillis(); SimpleResult results = inTransaction((conn) -> { SimpleResult dbVals = new SimpleResult(); try (PreparedStatement stmt = conn.prepareStatement(UPDATE_KIT_DEACTIVATION)) { - stmt.setLong(1, System.currentTimeMillis()); - stmt.setString(2, reason); + stmt.setLong(1, deactivatedDate); + stmt.setString(2, deactivationReason); stmt.setString(3, userId); - stmt.setLong(4, kitRequestId); + stmt.setLong(4, dsmKitRequestId); int result = stmt.executeUpdate(); if (result == 1) { - logger.info("Deactivated kitRequest w/ dsm_kit_request_id " + kitRequestId); + logger.info("Deactivated kitRequest w/ dsm_kit_request_id " + dsmKitRequestId); } else { - throw new RuntimeException("Error setting kitRequest " + kitRequestId + " to deactivated. It was updating " + result + " rows"); + throw new RuntimeException("Error setting kitRequest " + dsmKitRequestId + " to deactivated. It was updating " + result + " rows"); } } catch (SQLException ex) { @@ -762,24 +765,23 @@ public static void deactivateKitRequest(long kitRequestId, @NonNull String reaso }); if (results.resultException != null) { - throw new RuntimeException("Error setting kitRequest to deactivated w/ dsm_kit_request_id " + kitRequestId, results.resultException); + throw new RuntimeException("Error setting kitRequest to deactivated w/ dsm_kit_request_id " + dsmKitRequestId, results.resultException); } if (Objects.nonNull(ddpInstanceDto)) { - new KitRequestShipping() - KitRequestShipping kitRequestShipping = new KitRequestShipping(ddpParticipantId, collaboratorPatientId, collaboratorSampleId, null, - null, null, Long.parseLong(ddpKitRequestId), 0, null, null, null, null, null, null, 0, false, errorMessage, 0, null, - 0, null, null, false, null, 0, null, externalOrderNumber, false, null, createdBy, null, - null, null, 0, false, uploadReason, null, null, null); + KitRequestShipping kitRequestShipping = new KitRequestShipping(dsmKitRequestId, null, null, null, null, null); + kitRequestShipping.setDeactivationReason(deactivationReason); + kitRequestShipping.setDeactivatedDate(deactivatedDate); Generator paramsGenerator = new ParamsGenerator(kitRequestShipping, ddpInstanceDto.getInstanceName()); - RequestPayload requestPayload = new RequestPayload(ddpInstanceDto.getEsParticipantIndex(), ddpParticipantId); - ScriptBuilder scriptBuilder = new NestedScriptBuilder("kitRequestShipping", "ddpKitRequestId"); - UpsertPainless upsertPainless = new UpsertPainless(paramsGenerator, requestPayload, scriptBuilder); + ScriptBuilder scriptBuilder = new NestedScriptBuilder(paramsGenerator.getPropertyName(), "dsmKitRequestId"); + MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("dsmKitRequestId", dsmKitRequestId); + NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("dsm.kitRequestShipping", matchQueryBuilder, ScoreMode.Avg); + UpsertPainless upsertPainless = new UpsertPainless(paramsGenerator, ddpInstanceDto.getEsParticipantIndex(), scriptBuilder, nestedQueryBuilder); upsertPainless.export(); } else { if (easypostApiKey != null) { - KitRequestShipping.refundKit(kitRequestId, easypostApiKey); + KitRequestShipping.refundKit(dsmKitRequestId, easypostApiKey, ddpInstanceDto); } } } @@ -885,20 +887,21 @@ public static String writeRequest(@NonNull String instanceId, @NonNull String dd return dbVals; }); + if (results.resultException != null) { throw new RuntimeException("Error adding kit request w/ ddpKitRequestId " + ddpKitRequestId, results.resultException); } if (Objects.nonNull(ddpInstance)) { KitRequestShipping kitRequestShipping = new KitRequestShipping(ddpParticipantId, collaboratorPatientId, collaboratorSampleId, null, - null, null, Long.parseLong(ddpKitRequestId), 0, null, null, null, null, null, null, 0, false, errorMessage, 0, null, + null, null, (long) results.resultValue, 0, null, null, null, null, null, null, 0, false, errorMessage, 0, null, 0, null, null, false, null, 0, null, externalOrderNumber, false, null, createdBy, null, null, null, 0, false, uploadReason, null, null, null); Generator paramsGenerator = new ParamsGenerator(kitRequestShipping, ddpInstance.getName()); - RequestPayload requestPayload = new RequestPayload(ddpInstance.getParticipantIndexES(), ddpParticipantId); - ScriptBuilder scriptBuilder = new NestedScriptBuilder("kitRequestShipping", "ddpKitRequestId"); - UpsertPainless upsertPainless = new UpsertPainless(paramsGenerator, requestPayload, scriptBuilder); + ScriptBuilder scriptBuilder = new NestedScriptBuilder(paramsGenerator.getPropertyName(), "dsmKitRequestId"); + MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("_id", ddpParticipantId); + UpsertPainless upsertPainless = new UpsertPainless(paramsGenerator, ddpInstance.getParticipantIndexES(), scriptBuilder, matchQueryBuilder); upsertPainless.export(); } @@ -1289,7 +1292,7 @@ private static Shipment getEasyPostShipment(@NonNull EasyPostUtil easyPostUtil, return null; } - public static void refundKit(@NonNull long kitRequestId, @NonNull String easypostApiKey) { + public static void refundKit(long kitRequestId, @NonNull String easypostApiKey, DDPInstanceDto ddpInstanceDto) { KitShippingIds shippingIds = KitShippingIds.getKitShippingIds(kitRequestId, easypostApiKey); if (shippingIds != null) { String message = ""; @@ -1314,24 +1317,24 @@ public static void refundKit(@NonNull long kitRequestId, @NonNull String easypos } } if (StringUtils.isNotBlank(message)) { - updateKitError(kitRequestId, message); + updateKitError(kitRequestId, message, ddpInstanceDto); } } } - public static void updateKitError(@NonNull long kitRequestId, @NonNull String message) { + public static void updateKitError(@NonNull long dsmKitRequestId, @NonNull String message, DDPInstanceDto ddpInstanceDto) { SimpleResult results = inTransaction((conn) -> { SimpleResult dbVals = new SimpleResult(); try (PreparedStatement stmt = conn.prepareStatement(TransactionWrapper.getSqlFromConfig(ApplicationConfigConstants.UPDATE_KIT_ERROR))) { stmt.setInt(1, 1); stmt.setString(2, message); - stmt.setLong(3, kitRequestId); + stmt.setLong(3, dsmKitRequestId); int result = stmt.executeUpdate(); if (result == 1) { - logger.info("Updated error/message for kit request w/ dsm_kit_request_id " + kitRequestId); + logger.info("Updated error/message for kit request w/ dsm_kit_request_id " + dsmKitRequestId); } else { - throw new RuntimeException("Error updating error/message of kitRequest " + kitRequestId + ". It was updating " + result + " rows"); + throw new RuntimeException("Error updating error/message of kitRequest " + dsmKitRequestId + ". It was updating " + result + " rows"); } } catch (SQLException ex) { @@ -1341,26 +1344,35 @@ public static void updateKitError(@NonNull long kitRequestId, @NonNull String me }); if (results.resultException != null) { - throw new RuntimeException("Error updating error/message of kitRequest w/ dsm_kit_request_id " + kitRequestId, results.resultException); + throw new RuntimeException("Error updating error/message of kitRequest w/ dsm_kit_request_id " + dsmKitRequestId, results.resultException); } + + KitRequestShipping kitRequestShipping = new KitRequestShipping(dsmKitRequestId, null, null, null, null, message); + Generator paramsGenerator = new ParamsGenerator(kitRequestShipping, ddpInstanceDto.getInstanceName()); + ScriptBuilder scriptBuilder = new NestedScriptBuilder(paramsGenerator.getPropertyName(), "dsmKitRequestId"); + MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("dsmKitRequestId", dsmKitRequestId); + NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("dsm.kitRequestShipping", matchQueryBuilder, ScoreMode.Avg); + UpsertPainless upsertPainless = new UpsertPainless(paramsGenerator, ddpInstanceDto.getEsParticipantIndex(), scriptBuilder, nestedQueryBuilder); + upsertPainless.export(); + } public static void reactivateKitRequest(@NonNull String kitRequestId) { reactivateKitRequest(kitRequestId, null); } - public static void reactivateKitRequest(@NonNull String kitRequestId, String message) { + public static void reactivateKitRequest(@NonNull String dsmKitRequestId, String message) { SimpleResult results = inTransaction((conn) -> { SimpleResult dbVals = new SimpleResult(); try (PreparedStatement stmt = conn.prepareStatement(SQL_SELECT_KIT + QueryExtension.KIT_DEACTIVATED, ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_READ_ONLY)) { - stmt.setString(1, kitRequestId); + stmt.setString(1, dsmKitRequestId); try (ResultSet rs = stmt.executeQuery()) { rs.last(); int count = rs.getRow(); rs.beforeFirst(); if (count != 1) { - throw new RuntimeException("Couldn't find kit w/ dsm_kit_request_id " + kitRequestId + ". Rowcount: " + count); + throw new RuntimeException("Couldn't find kit w/ dsm_kit_request_id " + dsmKitRequestId + ". Rowcount: " + count); } if (rs.next()) { dbVals.resultValue = new KitRequestShipping( @@ -1380,11 +1392,11 @@ public static void reactivateKitRequest(@NonNull String kitRequestId, String mes }); if (results.resultException != null) { - throw new RuntimeException("Error reactivating kit request w/ dsm_kit_request_id " + kitRequestId, results.resultException); + throw new RuntimeException("Error reactivating kit request w/ dsm_kit_request_id " + dsmKitRequestId, results.resultException); } if (results.resultValue != null) { KitRequestShipping kitRequestShipping = (KitRequestShipping) results.resultValue; - KitRequestShipping.writeNewKit(kitRequestId, kitRequestShipping.getEasypostAddressId(), message, false); + KitRequestShipping.writeNewKit(dsmKitRequestId, kitRequestShipping.getEasypostAddressId(), message, false); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/UpsertPainless.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/UpsertPainless.java index 06777a226..437bcbeb3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/UpsertPainless.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/UpsertPainless.java @@ -1,39 +1,41 @@ package org.broadinstitute.dsm.model.elastic.export.painless; -import java.io.IOException; - -import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.Exportable; import org.broadinstitute.dsm.model.elastic.export.RequestPayload; import org.broadinstitute.dsm.model.elastic.export.generate.Generator; import org.broadinstitute.dsm.util.ElasticSearchUtil; -import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.reindex.UpdateByQueryRequest; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; +import java.io.IOException; + public class UpsertPainless implements Exportable { private Generator generator; - private RequestPayload requestPayload; // index, docId + private String index; // index, docId private ScriptBuilder scriptBuilder; + private QueryBuilder queryBuilder; - public UpsertPainless(Generator generator, RequestPayload requestPayload, ScriptBuilder scriptBuilder) { + public UpsertPainless(Generator generator, String index, ScriptBuilder scriptBuilder, QueryBuilder queryBuilder) { this.generator = generator; - this.requestPayload = requestPayload; + this.index = index; this.scriptBuilder = scriptBuilder; + this.queryBuilder = queryBuilder; } @Override public void export() { RestHighLevelClient clientInstance = ElasticSearchUtil.getClientInstance(); - UpdateRequest updateRequest = new UpdateRequest(requestPayload.getIndex(), Util.DOC, requestPayload.getDocId()); Script painless = new Script(ScriptType.INLINE, "painless", scriptBuilder.build(), generator.generate()); - updateRequest.script(painless); + UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(index); + updateByQueryRequest.setQuery(queryBuilder); + updateByQueryRequest.setScript(painless); try { - clientInstance.update(updateRequest, RequestOptions.DEFAULT); + clientInstance.updateByQuery(updateByQueryRequest, RequestOptions.DEFAULT); } catch (IOException e) { throw new RuntimeException("Error occurred while exporting data to ES", e); } diff --git a/src/main/java/org/broadinstitute/dsm/route/KitExpressRoute.java b/src/main/java/org/broadinstitute/dsm/route/KitExpressRoute.java index a031a6563..63dc62705 100644 --- a/src/main/java/org/broadinstitute/dsm/route/KitExpressRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/KitExpressRoute.java @@ -78,6 +78,7 @@ public void expressKitRequest(@NonNull String kitRequestId, @NonNull String user DDPInstance ddpInstance = DDPInstance.getDDPInstance(kitRequest.getRealm()); + EasyPostUtil easyPostUtil = new EasyPostUtil(kitRequest.getRealm()); HashMap kitTypes = org.broadinstitute.dsm.model.KitType.getKitLookup(); String key = kitRequest.getKitTypeName() + "_" + ddpInstance.getDdpInstanceId(); diff --git a/src/main/java/org/broadinstitute/dsm/route/ParticipantExitRoute.java b/src/main/java/org/broadinstitute/dsm/route/ParticipantExitRoute.java index 74b02ec1f..6ec98040b 100644 --- a/src/main/java/org/broadinstitute/dsm/route/ParticipantExitRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/ParticipantExitRoute.java @@ -11,6 +11,8 @@ import org.broadinstitute.dsm.db.KitDiscard; import org.broadinstitute.dsm.db.KitRequestShipping; import org.broadinstitute.dsm.db.ParticipantExit; +import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; +import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; import org.broadinstitute.dsm.exception.ParticipantNotExist; import org.broadinstitute.dsm.security.RequestHandler; import org.broadinstitute.dsm.statics.DBConstants; @@ -26,6 +28,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Optional; public class ParticipantExitRoute extends RequestHandler { @@ -66,7 +69,8 @@ public Object processRequest(Request request, Response response, String userId) } else { //refund label of kits which are not sent yet - KitRequestShipping.refundKit(kit.getDsmKitRequestId(), DSMServer.getDDPEasypostApiKey(realm)); + Optional maybeInstance = new DDPInstanceDao().getDDPInstanceByInstanceName(realm); + KitRequestShipping.refundKit(kit.getDsmKitRequestId(), DSMServer.getDDPEasypostApiKey(realm), maybeInstance.orElse(null)); } } return kitsNeedAction; From a1802779d935b2e40f56fbdbf2b3e590ec85ae67 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 24 Jan 2022 11:51:02 +0400 Subject: [PATCH 337/405] DDP-7443 | add script for reactivate kit --- .../dsm/db/KitRequestShipping.java | 31 +++++++++++++------ .../dsm/route/KitDeactivationRoute.java | 11 ++++--- .../dsm/route/KitExpressRoute.java | 23 ++++++++------ 3 files changed, 42 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java index 1034ed207..e037bbc41 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java @@ -911,7 +911,7 @@ public static String writeRequest(@NonNull String instanceId, @NonNull String dd private static SimpleResult writeNewKit(Connection conn, String kitRequestId, String addressIdTo, String errorMessage, boolean needsApproval) { SimpleResult dbVals = new SimpleResult(); - try (PreparedStatement insertKit = conn.prepareStatement(INSERT_KIT)) { + try (PreparedStatement insertKit = conn.prepareStatement(INSERT_KIT, Statement.RETURN_GENERATED_KEYS)) { insertKit.setString(1, kitRequestId); if (StringUtils.isNotBlank(addressIdTo)) { insertKit.setString(2, addressIdTo); @@ -928,6 +928,14 @@ private static SimpleResult writeNewKit(Connection conn, String kitRequestId, St insertKit.setObject(4, errorMessage); insertKit.setBoolean(5, needsApproval); insertKit.executeUpdate(); + try (ResultSet rs = insertKit.getGeneratedKeys()) { + if (rs.next()) { + dbVals.resultValue = rs.getString(1); + } + } + catch (Exception e) { + throw new RuntimeException("Error getting id of new kit request ", e); + } } catch (SQLException e) { dbVals.resultException = e; @@ -936,10 +944,8 @@ private static SimpleResult writeNewKit(Connection conn, String kitRequestId, St } // called by reactivation of a deactivated kit - public static void writeNewKit(String kitRequestId, String addressIdTo, String errorMessage, boolean needsApproval) { - SimpleResult results = inTransaction((conn) -> { - return writeNewKit(conn, kitRequestId, addressIdTo, errorMessage, needsApproval); - }); + public static long writeNewKit(String kitRequestId, String addressIdTo, String errorMessage, boolean needsApproval) { + SimpleResult results = inTransaction((conn) -> writeNewKit(conn, kitRequestId, addressIdTo, errorMessage, needsApproval)); if (results.resultException != null) { logger.error("Error writing new kit w/ dsm_kit_id " + kitRequestId, results.resultException); @@ -947,6 +953,7 @@ public static void writeNewKit(String kitRequestId, String addressIdTo, String e else { logger.info("Wrote new kit w/ dsm_kit_id " + kitRequestId, results.resultException); } + return (long) results.resultValue; } // update kit with label trigger user and date @@ -1357,11 +1364,11 @@ public static void updateKitError(@NonNull long dsmKitRequestId, @NonNull String } - public static void reactivateKitRequest(@NonNull String kitRequestId) { - reactivateKitRequest(kitRequestId, null); + public static void reactivateKitRequest(@NonNull String kitRequestId, DDPInstanceDto ddpInstanceDto) { + reactivateKitRequest(kitRequestId, null, ddpInstanceDto); } - public static void reactivateKitRequest(@NonNull String dsmKitRequestId, String message) { + public static void reactivateKitRequest(@NonNull String dsmKitRequestId, String message, DDPInstanceDto ddpInstanceDto) { SimpleResult results = inTransaction((conn) -> { SimpleResult dbVals = new SimpleResult(); try (PreparedStatement stmt = conn.prepareStatement(SQL_SELECT_KIT + QueryExtension.KIT_DEACTIVATED, @@ -1396,7 +1403,13 @@ public static void reactivateKitRequest(@NonNull String dsmKitRequestId, String } if (results.resultValue != null) { KitRequestShipping kitRequestShipping = (KitRequestShipping) results.resultValue; - KitRequestShipping.writeNewKit(dsmKitRequestId, kitRequestShipping.getEasypostAddressId(), message, false); + long dsmKitId = KitRequestShipping.writeNewKit(dsmKitRequestId, kitRequestShipping.getEasypostAddressId(), message, false); + Generator paramsGenerator = new ParamsGenerator(kitRequestShipping, ddpInstanceDto.getInstanceName()); + ScriptBuilder scriptBuilder = new NestedScriptBuilder(paramsGenerator.getPropertyName(), "dsmKitId"); + MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("dsmKitId", dsmKitId); + NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("dsm.kitRequestShipping", matchQueryBuilder, ScoreMode.Avg); + UpsertPainless upsertPainless = new UpsertPainless(paramsGenerator, ddpInstanceDto.getEsParticipantIndex(), scriptBuilder, nestedQueryBuilder); + upsertPainless.export(); } } diff --git a/src/main/java/org/broadinstitute/dsm/route/KitDeactivationRoute.java b/src/main/java/org/broadinstitute/dsm/route/KitDeactivationRoute.java index 3f5f31736..7e6567deb 100644 --- a/src/main/java/org/broadinstitute/dsm/route/KitDeactivationRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/KitDeactivationRoute.java @@ -47,13 +47,13 @@ public Object processRequest(Request request, Response response, String userId) boolean deactivate = request.url().toLowerCase().contains("deactivate"); KitRequestShipping kitRequest = KitRequestShipping.getKitRequest(kitRequestId); String realm = kitRequest.getRealm(); - Optional ddpInstanceByInstanceName = new DDPInstanceDao().getDDPInstanceByInstanceName(realm); + DDPInstanceDto ddpInstanceByInstanceName = new DDPInstanceDao().getDDPInstanceByInstanceName(realm).orElse(null); if ( UserUtil.checkUserAccess(realm, userId, "kit_deactivation", userIdRequest)) { if (deactivate) { JsonObject jsonObject = new JsonParser().parse(request.body()).getAsJsonObject(); String reason = jsonObject.get("reason").getAsString(); KitRequestShipping.deactivateKitRequest(Long.parseLong(kitRequestId), reason, - DSMServer.getDDPEasypostApiKey(realm), userIdRequest, ddpInstanceByInstanceName.orElse(null)); + DSMServer.getDDPEasypostApiKey(realm), userIdRequest, ddpInstanceByInstanceName); } else { QueryParamsMap queryParams = request.queryMap(); @@ -62,7 +62,8 @@ public Object processRequest(Request request, Response response, String userId) activateAnyway = queryParams.get("activate").booleanValue(); } if (activateAnyway) { - KitRequestShipping.reactivateKitRequest(kitRequestId, KitUtil.IGNORE_AUTO_DEACTIVATION); + KitRequestShipping.reactivateKitRequest(kitRequestId, KitUtil.IGNORE_AUTO_DEACTIVATION, + ddpInstanceByInstanceName); } else { DDPInstance ddpInstance = DDPInstance.getDDPInstance(realm); @@ -91,11 +92,11 @@ else if (InstanceSettings.TYPE_NOTIFICATION.equals(activation.getType())) { } } else { - KitRequestShipping.reactivateKitRequest(kitRequestId); + KitRequestShipping.reactivateKitRequest(kitRequestId, ddpInstanceByInstanceName); } } else { - KitRequestShipping.reactivateKitRequest(kitRequestId); + KitRequestShipping.reactivateKitRequest(kitRequestId, ddpInstanceByInstanceName); } } diff --git a/src/main/java/org/broadinstitute/dsm/route/KitExpressRoute.java b/src/main/java/org/broadinstitute/dsm/route/KitExpressRoute.java index 63dc62705..9c6037c32 100644 --- a/src/main/java/org/broadinstitute/dsm/route/KitExpressRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/KitExpressRoute.java @@ -12,6 +12,8 @@ import org.broadinstitute.dsm.DSMServer; import org.broadinstitute.dsm.db.DDPInstance; import org.broadinstitute.dsm.db.KitRequestShipping; +import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; +import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; import org.broadinstitute.dsm.model.EasypostLabelRate; import org.broadinstitute.dsm.model.KitRequestSettings; import org.broadinstitute.dsm.model.KitType; @@ -70,30 +72,33 @@ public Object processRequest(Request request, Response response, String userId) public void expressKitRequest(@NonNull String kitRequestId, @NonNull String userId) { KitRequestShipping kitRequest = KitRequestShipping.getKitRequest(kitRequestId); + + DDPInstanceDto ddpInstanceDto = new DDPInstanceDao().getDDPInstanceByInstanceName(kitRequest.getRealm()).orElseThrow(); + //deactivate kit which is already in db and refund the label - KitRequestShipping.deactivateKitRequest(Long.valueOf(kitRequestId), KitRequestShipping.DEACTIVATION_REASON, - DSMServer.getDDPEasypostApiKey(kitRequest.getRealm()), userId, ddpInstanceByInstanceName.orElse(null)); + KitRequestShipping.deactivateKitRequest(Long.parseLong(kitRequestId), KitRequestShipping.DEACTIVATION_REASON, + DSMServer.getDDPEasypostApiKey(kitRequest.getRealm()), userId, ddpInstanceDto); //add new kit into db - KitRequestShipping.reactivateKitRequest(kitRequestId); + KitRequestShipping.reactivateKitRequest(kitRequestId, ddpInstanceDto); - DDPInstance ddpInstance = DDPInstance.getDDPInstance(kitRequest.getRealm()); EasyPostUtil easyPostUtil = new EasyPostUtil(kitRequest.getRealm()); HashMap kitTypes = org.broadinstitute.dsm.model.KitType.getKitLookup(); - String key = kitRequest.getKitTypeName() + "_" + ddpInstance.getDdpInstanceId(); + String key = kitRequest.getKitTypeName() + "_" + ddpInstanceDto.getDdpInstanceId(); KitType kitType = kitTypes.get(key); - Map carrierServiceTypes = KitRequestSettings.getKitRequestSettings(ddpInstance.getDdpInstanceId()); + Map carrierServiceTypes = + KitRequestSettings.getKitRequestSettings(String.valueOf(ddpInstanceDto.getDdpInstanceId())); KitRequestSettings kitRequestSettings = carrierServiceTypes.get(kitType.getKitTypeId()); String kitId = getKitId(kitRequestId); //trigger label creation - createExpressLabelToParticipant(easyPostUtil, kitRequestSettings, kitType, kitId, kitRequest.getEasypostAddressId(), ddpInstance); + createExpressLabelToParticipant(easyPostUtil, kitRequestSettings, kitType, kitId, kitRequest.getEasypostAddressId(), ddpInstanceDto); } private void createExpressLabelToParticipant(@NonNull EasyPostUtil easyPostUtil, @NonNull KitRequestSettings kitRequestSettings, - @NonNull KitType kitType, @NonNull String kitId, @NonNull String addressToId, @NonNull DDPInstance ddpInstance) { + @NonNull KitType kitType, @NonNull String kitId, @NonNull String addressToId, DDPInstanceDto ddpInstance) { String errorMessage = ""; Shipment participantShipment = null; Address toAddress = null; @@ -101,7 +106,7 @@ private void createExpressLabelToParticipant(@NonNull EasyPostUtil easyPostUtil, toAddress = KitRequestShipping.getToAddressId(easyPostUtil, kitRequestSettings, addressToId, null); participantShipment = KitRequestShipping.getShipment(easyPostUtil, ddpInstance.getBillingReference(), kitType, kitRequestSettings, toAddress, "FedEx", kitRequestSettings.getCarrierToId(), "FIRST_OVERNIGHT"); - doNotification(ddpInstance.getName()); + doNotification(ddpInstance.getInstanceName()); } catch (Exception e) { errorMessage = "To: " + e.getMessage(); From e621a66fbf10613ba0a4c093d3abfaae5553dadb Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 24 Jan 2022 16:39:45 +0400 Subject: [PATCH 338/405] DDP-7443 | start adding exportToES() to updateKit() --- .../dsm/db/KitRequestCreateLabel.java | 3 +- .../dsm/db/KitRequestShipping.java | 58 +++++++++++-------- .../dsm/model/elastic/export/Exportable.java | 2 +- .../export/painless/NestedScriptBuilder.java | 3 +- .../model/elastic/migration/BaseMigrator.java | 3 +- .../dsm/route/KitLabelRoute.java | 18 ++++-- 6 files changed, 55 insertions(+), 32 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/KitRequestCreateLabel.java b/src/main/java/org/broadinstitute/dsm/db/KitRequestCreateLabel.java index cd981e7a3..d66528e46 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitRequestCreateLabel.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitRequestCreateLabel.java @@ -4,6 +4,7 @@ import lombok.NonNull; import org.apache.commons.lang3.StringUtils; import org.broadinstitute.ddp.db.SimpleResult; +import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; import org.broadinstitute.dsm.model.KitRequestSettings; import org.broadinstitute.dsm.model.KitType; import org.broadinstitute.dsm.statics.DBConstants; @@ -66,7 +67,7 @@ public KitRequestCreateLabel(@NonNull String dsmKitId, @NonNull String dsmKitReq /** * Set given kitRequests to need label */ - public static void updateKitLabelRequested(KitRequestShipping[] kitRequests, @NonNull String userId) { + public static void updateKitLabelRequested(KitRequestShipping[] kitRequests, @NonNull String userId, DDPInstanceDto ddpInstanceDto) { if (kitRequests.length > 0) { for (KitRequestShipping kitRequest : kitRequests) { KitRequestShipping.updateKit(kitRequest.getDsmKitId(), userId); diff --git a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java index e037bbc41..b0508c126 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java @@ -20,6 +20,7 @@ import org.broadinstitute.dsm.model.*; import org.broadinstitute.dsm.model.ddp.DDPParticipant; import org.broadinstitute.dsm.model.ddp.KitDetail; +import org.broadinstitute.dsm.model.elastic.export.Exportable; import org.broadinstitute.dsm.model.elastic.export.generate.Generator; import org.broadinstitute.dsm.model.elastic.export.painless.NestedScriptBuilder; import org.broadinstitute.dsm.model.elastic.export.painless.ParamsGenerator; @@ -208,7 +209,8 @@ public class KitRequestShipping extends KitRequest { @ColumnName (DBConstants.EASYPOST_TO_ID) private String easypostToId; - private Long labelTriggeredDate; + @ColumnName (DBConstants.LABEL_TRIGGERED_DATE) + private Long labelDate; @ColumnName (DBConstants.NO_RETURN) private Boolean noReturn; @@ -275,7 +277,7 @@ public KitRequestShipping(String participantId, String collaboratorParticipantId String trackingNumberTo, String trackingNumberReturn, String trackingUrlTo, String trackingUrlReturn, long scanDate, boolean error, String message, long receiveDate, String easypostAddressId, long deactivatedDate, String deactivationReason, - String kitLabel, boolean express, String easypostToId, long labelTriggeredDate, String easypostShipmentStatus, + String kitLabel, boolean express, String easypostToId, long labelDate, String easypostShipmentStatus, String externalOrderNumber, boolean noReturn, String externalOrderStatus, String createdBy, String testResult, String upsTrackingStatus, String upsReturnStatus, long externalOrderDate, boolean careEvolve, String uploadReason, String receiveDateString, String hruid, String gender) { @@ -301,7 +303,7 @@ public KitRequestShipping(String participantId, String collaboratorParticipantId this.kitLabel = kitLabel; this.express = express; this.easypostToId = easypostToId; - this.labelTriggeredDate = labelTriggeredDate; + this.labelDate = labelDate; this.easypostShipmentStatus = easypostShipmentStatus; this.noReturn = noReturn; this.createdBy = createdBy; @@ -772,12 +774,8 @@ public static void deactivateKitRequest(long dsmKitRequestId, @NonNull String de KitRequestShipping kitRequestShipping = new KitRequestShipping(dsmKitRequestId, null, null, null, null, null); kitRequestShipping.setDeactivationReason(deactivationReason); kitRequestShipping.setDeactivatedDate(deactivatedDate); - Generator paramsGenerator = new ParamsGenerator(kitRequestShipping, ddpInstanceDto.getInstanceName()); - ScriptBuilder scriptBuilder = new NestedScriptBuilder(paramsGenerator.getPropertyName(), "dsmKitRequestId"); - MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("dsmKitRequestId", dsmKitRequestId); - NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("dsm.kitRequestShipping", matchQueryBuilder, ScoreMode.Avg); - UpsertPainless upsertPainless = new UpsertPainless(paramsGenerator, ddpInstanceDto.getEsParticipantIndex(), scriptBuilder, nestedQueryBuilder); - upsertPainless.export(); + + exportToES(kitRequestShipping, ddpInstanceDto, "dsmKitRequestId", "dsmKitRequestId", dsmKitRequestId); } else { if (easypostApiKey != null) { @@ -900,7 +898,7 @@ public static String writeRequest(@NonNull String instanceId, @NonNull String dd Generator paramsGenerator = new ParamsGenerator(kitRequestShipping, ddpInstance.getName()); ScriptBuilder scriptBuilder = new NestedScriptBuilder(paramsGenerator.getPropertyName(), "dsmKitRequestId"); - MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("_id", ddpParticipantId); + MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("_id", Exportable.getParticipantGuid(ddpParticipantId, ddpInstance.getParticipantIndexES())); UpsertPainless upsertPainless = new UpsertPainless(paramsGenerator, ddpInstance.getParticipantIndexES(), scriptBuilder, matchQueryBuilder); upsertPainless.export(); } @@ -958,11 +956,12 @@ public static long writeNewKit(String kitRequestId, String addressIdTo, String e // update kit with label trigger user and date public static void updateKit(long dsmKitId, String userId) { + long labelDate = System.currentTimeMillis(); SimpleResult results = inTransaction((conn) -> { SimpleResult dbVals = new SimpleResult(); try (PreparedStatement stmt = conn.prepareStatement(SQL_UPDATE_KIT)) { stmt.setString(1, userId); - stmt.setLong(2, System.currentTimeMillis()); + stmt.setLong(2, labelDate); stmt.setLong(3, dsmKitId); int result = stmt.executeUpdate(); @@ -984,6 +983,17 @@ public static void updateKit(long dsmKitId, String userId) { } else { logger.info("Updated kit w/ kit_id " + dsmKitId, results.resultException); + + KitRequestShipping kitRequestShipping = new KitRequestShipping(null, dsmKitId, null, null, null, null); + kitRequestShipping.setLabelDate(labelDate); + + exportToES(kitRequestShipping, ); + Generator paramsGenerator = new ParamsGenerator(kitRequestShipping, ddpInstanceDto.getInstanceName()); + ScriptBuilder scriptBuilder = new NestedScriptBuilder(paramsGenerator.getPropertyName(), "dsmKitId"); + MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("dsmKitRequestId", dsmKitRequestId); + NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("dsm.kitRequestShipping", matchQueryBuilder, ScoreMode.Avg); + UpsertPainless upsertPainless = new UpsertPainless(paramsGenerator, ddpInstanceDto.getEsParticipantIndex(), scriptBuilder, nestedQueryBuilder); + upsertPainless.export(); } } @@ -1355,12 +1365,7 @@ public static void updateKitError(@NonNull long dsmKitRequestId, @NonNull String } KitRequestShipping kitRequestShipping = new KitRequestShipping(dsmKitRequestId, null, null, null, null, message); - Generator paramsGenerator = new ParamsGenerator(kitRequestShipping, ddpInstanceDto.getInstanceName()); - ScriptBuilder scriptBuilder = new NestedScriptBuilder(paramsGenerator.getPropertyName(), "dsmKitRequestId"); - MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("dsmKitRequestId", dsmKitRequestId); - NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("dsm.kitRequestShipping", matchQueryBuilder, ScoreMode.Avg); - UpsertPainless upsertPainless = new UpsertPainless(paramsGenerator, ddpInstanceDto.getEsParticipantIndex(), scriptBuilder, nestedQueryBuilder); - upsertPainless.export(); + exportToES(kitRequestShipping, ddpInstanceDto, "dsmKitRequestId", "dsmKitRequestId", dsmKitRequestId); } @@ -1403,16 +1408,23 @@ public static void reactivateKitRequest(@NonNull String dsmKitRequestId, String } if (results.resultValue != null) { KitRequestShipping kitRequestShipping = (KitRequestShipping) results.resultValue; + long dsmKitId = KitRequestShipping.writeNewKit(dsmKitRequestId, kitRequestShipping.getEasypostAddressId(), message, false); - Generator paramsGenerator = new ParamsGenerator(kitRequestShipping, ddpInstanceDto.getInstanceName()); - ScriptBuilder scriptBuilder = new NestedScriptBuilder(paramsGenerator.getPropertyName(), "dsmKitId"); - MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("dsmKitId", dsmKitId); - NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("dsm.kitRequestShipping", matchQueryBuilder, ScoreMode.Avg); - UpsertPainless upsertPainless = new UpsertPainless(paramsGenerator, ddpInstanceDto.getEsParticipantIndex(), scriptBuilder, nestedQueryBuilder); - upsertPainless.export(); + kitRequestShipping.setDsmKitId(dsmKitId); + + exportToES(kitRequestShipping, ddpInstanceDto, "dsmKitId", "dsmKitRequestId", dsmKitRequestId); } } + private static void exportToES(KitRequestShipping kitRequestShipping, DDPInstanceDto ddpInstanceDto, String uniqueIdentifier, String fieldName, Object fieldValue) { + Generator paramsGenerator = new ParamsGenerator(kitRequestShipping, ddpInstanceDto.getInstanceName()); + ScriptBuilder scriptBuilder = new NestedScriptBuilder(paramsGenerator.getPropertyName(), uniqueIdentifier); + MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder(fieldName, fieldValue); + NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("dsm.kitRequestShipping", matchQueryBuilder, ScoreMode.Avg); + UpsertPainless upsertPainless = new UpsertPainless(paramsGenerator, ddpInstanceDto.getEsParticipantIndex(), scriptBuilder, nestedQueryBuilder); + upsertPainless.export(); + } + public static List findKitRequest(@NonNull String field, @NonNull String value, String[] realms) { Map kitRequests = new HashMap<>(); SimpleResult results = inTransaction((conn) -> { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java index efaeb8d0a..86bc821a2 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/Exportable.java @@ -8,7 +8,7 @@ public interface Exportable { void export(); - default String getParticipantGuid(String participantId, String index) { + static String getParticipantGuid(String participantId, String index) { if (!(ParticipantUtil.isGuid(participantId))) { ElasticSearchParticipantDto participantById = new ElasticSearch().getParticipantById(index, participantId); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedScriptBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedScriptBuilder.java index 14535d8bc..2699cf56a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedScriptBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedScriptBuilder.java @@ -4,7 +4,8 @@ public class NestedScriptBuilder implements ScriptBuilder { private final static String SCRIPT = "if (ctx._source.dsm.#propertyName == null) {ctx._source.dsm.#propertyName = [params.dsm.#propertyName]} " + "else {def targets = ctx._source.dsm.#propertyName.findAll(obj -> obj.#uniqueIdentifier == params.dsm.#propertyName.#uniqueIdentifier); " + - "for(target in targets) { for (entry in params.dsm.#propertyName.entrySet()) { target.put(entry.getKey(), entry.getValue()) } }}"; + "if (targets.size() == 0) { ctx._source.dsm.#propertyName.add(params.dsm.#propertyName) } " + + "else { for(target in targets) { for (entry in params.dsm.#propertyName.entrySet()) { target.put(entry.getKey(), entry.getValue()) } }}}"; private final String propertyName; private final String uniqueIdentifier; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseMigrator.java index e575ab45c..48533e470 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/BaseMigrator.java @@ -1,6 +1,7 @@ package org.broadinstitute.dsm.model.elastic.migration; import org.broadinstitute.dsm.model.elastic.export.BaseExporter; +import org.broadinstitute.dsm.model.elastic.export.Exportable; import org.broadinstitute.dsm.model.elastic.export.generate.Generator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,7 +29,7 @@ protected void fillBulkRequestWithTransformedMap(Map participant logger.info("filling bulk request with participants and their details"); for (Map.Entry entry: participantRecords.entrySet()) { String participantId = entry.getKey(); - participantId = getParticipantGuid(participantId, index); + participantId = Exportable.getParticipantGuid(participantId, index); if (StringUtils.isBlank(participantId)) continue; Object participantDetails = entry.getValue(); transformObject(participantDetails); diff --git a/src/main/java/org/broadinstitute/dsm/route/KitLabelRoute.java b/src/main/java/org/broadinstitute/dsm/route/KitLabelRoute.java index 484bfd783..493a5d6b8 100644 --- a/src/main/java/org/broadinstitute/dsm/route/KitLabelRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/KitLabelRoute.java @@ -4,6 +4,8 @@ import org.broadinstitute.ddp.handlers.util.Result; import org.broadinstitute.dsm.db.KitRequestCreateLabel; import org.broadinstitute.dsm.db.KitRequestShipping; +import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; +import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; import org.broadinstitute.dsm.security.RequestHandler; import org.broadinstitute.dsm.statics.RoutePath; import org.broadinstitute.dsm.statics.UserErrorMessages; @@ -14,6 +16,8 @@ import spark.Request; import spark.Response; +import java.util.Optional; + public class KitLabelRoute extends RequestHandler { @Override @@ -26,19 +30,23 @@ public Object processRequest(Request request, Response response, String userId) if (UserUtil.checkUserAccess(null, userId, "kit_shipping", userIdRequest)) { QueryParamsMap queryParams = request.queryMap(); + String realm = null; + if (queryParams.value(RoutePath.REALM) != null) { + realm = queryParams.get(RoutePath.REALM).value(); + } + + DDPInstanceDto ddpInstanceDto = new DDPInstanceDao().getDDPInstanceByInstanceName(realm) + .orElseThrow(); + String requestBody = request.body(); if (requestBody != null) { KitRequestShipping[] kitRequests = new Gson().fromJson(requestBody, KitRequestShipping[].class); if (kitRequests != null) { - KitRequestCreateLabel.updateKitLabelRequested(kitRequests, userIdRequest); + KitRequestCreateLabel.updateKitLabelRequested(kitRequests, userIdRequest, ddpInstanceDto); return new Result(200); } } - String realm = null; - if (queryParams.value(RoutePath.REALM) != null) { - realm = queryParams.get(RoutePath.REALM).value(); - } String kitType = null; if (queryParams.value(RoutePath.KIT_TYPE) != null) { kitType = queryParams.get(RoutePath.KIT_TYPE).value(); From 2c06c55688ece563b96ae22f36e4eb6246f48f12 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 24 Jan 2022 17:09:42 +0400 Subject: [PATCH 339/405] DDP-7443 | continue adding script to export to ES for kits --- .../dsm/db/KitRequestCreateLabel.java | 7 ++--- .../dsm/db/KitRequestShipping.java | 26 +++++++++---------- .../org/broadinstitute/dsm/util/KitUtil.java | 13 ++++++++-- 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/KitRequestCreateLabel.java b/src/main/java/org/broadinstitute/dsm/db/KitRequestCreateLabel.java index d66528e46..99db45363 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitRequestCreateLabel.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitRequestCreateLabel.java @@ -4,6 +4,7 @@ import lombok.NonNull; import org.apache.commons.lang3.StringUtils; import org.broadinstitute.ddp.db.SimpleResult; +import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; import org.broadinstitute.dsm.model.KitRequestSettings; import org.broadinstitute.dsm.model.KitType; @@ -70,7 +71,7 @@ public KitRequestCreateLabel(@NonNull String dsmKitId, @NonNull String dsmKitReq public static void updateKitLabelRequested(KitRequestShipping[] kitRequests, @NonNull String userId, DDPInstanceDto ddpInstanceDto) { if (kitRequests.length > 0) { for (KitRequestShipping kitRequest : kitRequests) { - KitRequestShipping.updateKit(kitRequest.getDsmKitId(), userId); + KitRequestShipping.updateKit(kitRequest.getDsmKitId(), userId, ddpInstanceDto); } logger.info("Done triggering label creation"); } @@ -110,10 +111,10 @@ public static void updateKitLabelRequested(String realm, String type, @NonNull S throw new RuntimeException("Error looking up the latestKitRequests ", results.resultException); } logger.info("Found " + kitsNoLabel.size() + " kit requests which need a label"); - + DDPInstanceDto ddpInstanceDto = new DDPInstanceDao().getDDPInstanceByInstanceName(realm).orElseThrow(); if (!kitsNoLabel.isEmpty()) { for (KitRequestCreateLabel uploadedKit : kitsNoLabel) { - KitRequestShipping.updateKit(Long.parseLong(uploadedKit.getDsmKitId()), userId); + KitRequestShipping.updateKit(Long.parseLong(uploadedKit.getDsmKitId()), userId, ddpInstanceDto); } } logger.info("Done triggering label creation"); diff --git a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java index b0508c126..ac044eba5 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java @@ -955,7 +955,7 @@ public static long writeNewKit(String kitRequestId, String addressIdTo, String e } // update kit with label trigger user and date - public static void updateKit(long dsmKitId, String userId) { + public static void updateKit(long dsmKitId, String userId, DDPInstanceDto ddpInstanceDto) { long labelDate = System.currentTimeMillis(); SimpleResult results = inTransaction((conn) -> { SimpleResult dbVals = new SimpleResult(); @@ -987,19 +987,14 @@ public static void updateKit(long dsmKitId, String userId) { KitRequestShipping kitRequestShipping = new KitRequestShipping(null, dsmKitId, null, null, null, null); kitRequestShipping.setLabelDate(labelDate); - exportToES(kitRequestShipping, ); - Generator paramsGenerator = new ParamsGenerator(kitRequestShipping, ddpInstanceDto.getInstanceName()); - ScriptBuilder scriptBuilder = new NestedScriptBuilder(paramsGenerator.getPropertyName(), "dsmKitId"); - MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("dsmKitRequestId", dsmKitRequestId); - NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("dsm.kitRequestShipping", matchQueryBuilder, ScoreMode.Avg); - UpsertPainless upsertPainless = new UpsertPainless(paramsGenerator, ddpInstanceDto.getEsParticipantIndex(), scriptBuilder, nestedQueryBuilder); - upsertPainless.export(); + exportToES(kitRequestShipping, ddpInstanceDto, "dsmKitId", "dsmKitId", dsmKitId); } } // update kit with label information - public static void updateKit(String kitRequestId, Shipment participantShipment, Shipment returnShipment, + public static void updateKit(String dsmKitId, Shipment participantShipment, Shipment returnShipment, String errorMessage, Address toAddress, boolean isExpress) { + KitRequestShipping kitRequestShipping = new KitRequestShipping(); SimpleResult results = inTransaction((conn) -> { SimpleResult dbVals = new SimpleResult(); try (PreparedStatement stmt = conn.prepareStatement(UPDATE_KIT)) { @@ -1010,6 +1005,10 @@ public static void updateKit(String kitRequestId, Shipment participantShipment, stmt.setString(3, participantShipment.getId()); stmt.setString(5, participantShipment.getTrackingCode()); stmt.setString(7, participantTracker.getPublicUrl()); + kitRequestShipping.setLabelUrlTo(participantLabel.getLabelUrl()); + kitRequestShipping.setEasypostToId(participantShipment.getId()); + kitRequestShipping.setTrackingNumberTo(participantShipment.getTrackingCode()); + kitRequestShipping.setTrackingUrlTo(participantTracker.getPublicUrl()); //TODO } else { stmt.setString(1, null); @@ -1024,6 +1023,7 @@ public static void updateKit(String kitRequestId, Shipment participantShipment, stmt.setString(4, returnShipment.getId()); stmt.setString(6, returnShipment.getTrackingCode()); stmt.setString(8, returnTracker.getPublicUrl()); + kitRequestShipping.setLabelUrlReturn(); } else { stmt.setString(2, null); @@ -1054,10 +1054,10 @@ public static void updateKit(String kitRequestId, Shipment participantShipment, else { stmt.setInt(12, 0); } - stmt.setString(13, kitRequestId); + stmt.setString(13, dsmKitId); int result = stmt.executeUpdate(); if (result != 1) { - throw new RuntimeException("Error updating kit " + kitRequestId + " it was updating " + result + " rows"); + throw new RuntimeException("Error updating kit " + dsmKitId + " it was updating " + result + " rows"); } } catch (Exception e) { @@ -1067,10 +1067,10 @@ public static void updateKit(String kitRequestId, Shipment participantShipment, }); if (results.resultException != null) { - logger.error("Error updating kit w/ dsm_kit_id " + kitRequestId, results.resultException); + logger.error("Error updating kit w/ dsm_kit_id " + dsmKitId, results.resultException); } else { - logger.info("Updated kit w/ dsm_kit_id " + kitRequestId, results.resultException); + logger.info("Updated kit w/ dsm_kit_id " + dsmKitId, results.resultException); } } diff --git a/src/main/java/org/broadinstitute/dsm/util/KitUtil.java b/src/main/java/org/broadinstitute/dsm/util/KitUtil.java index 01a4502e3..899a893fa 100644 --- a/src/main/java/org/broadinstitute/dsm/util/KitUtil.java +++ b/src/main/java/org/broadinstitute/dsm/util/KitUtil.java @@ -13,6 +13,8 @@ import org.broadinstitute.dsm.db.InstanceSettings; import org.broadinstitute.dsm.db.KitRequestShipping; import org.broadinstitute.dsm.db.KitRequestCreateLabel; +import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; +import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; import org.broadinstitute.dsm.model.KitRequestSettings; import org.broadinstitute.dsm.model.KitType; import org.broadinstitute.dsm.model.Value; @@ -128,8 +130,15 @@ public static void createLabel(List kitsLabelTriggered) { Address toAddress = null; try { if (StringUtils.isBlank(kitLabelTriggered.getAddressIdTo())) { + DDPInstance ddpInstance = DDPInstance.getDDPInstance(kitLabelTriggered.getInstanceName()); + //TODO -> before we finally switch to ddpInstanceDao/ddpInstanceDto pair + DDPInstanceDto ddpInstanceDto = new DDPInstanceDto.Builder() + .withInstanceName(ddpInstance.getName()) + .withEsParticipantIndex(ddpInstance.getParticipantIndexES()) + .build(); + Map> participantESData = ElasticSearchUtil.getFilteredDDPParticipantsFromES(ddpInstance, ElasticSearchUtil.BY_GUID + kitLabelTriggered.getDdpParticipantId()); if (participantESData == null || participantESData.isEmpty()) { @@ -143,7 +152,7 @@ public static void createLabel(List kitsLabelTriggered) { else { KitRequestShipping.deactivateKitRequest(Long.parseLong(kitLabelTriggered.getDsmKitRequestId()), "Participant not found", null, - "System", ddpInstanceByInstanceName.orElse(null)); + "System", ddpInstanceDto); logger.error("Didn't find participant " + kitLabelTriggered.getDdpParticipantId()); } } @@ -526,7 +535,7 @@ public static List findSpecialBehaviorKits(@NonNull Notifica if (specialBehavior) { KitRequestShipping.deactivateKitRequest(kit.getDsmKitRequestId(), SYSTEM_AUTOMATICALLY_DEACTIVATED + ": " + uploaded.getValue(), DSMServer.getDDPEasypostApiKey(ddpInstance.getName()), "System", - ddpInstanceByInstanceName.orElse(null)); + new DDPInstanceDao().getDDPInstanceByInstanceName(ddpInstance.getName()).orElseThrow()); if (InstanceSettings.TYPE_NOTIFICATION.equals(uploaded.getType())) { String message = kitType.getName() + " kit for participant " + kit.getParticipantId() + " (" + kit.getCollaboratorParticipantId() + ") was deactivated per background job
. " + uploaded.getValue(); From 2877eeed9714b1beff8f901d3f4cf963a044e0c8 Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 24 Jan 2022 17:38:07 +0400 Subject: [PATCH 340/405] DDP-7443 | add exportToEs to updateKit() methods --- .../dsm/db/KitRequestShipping.java | 27 ++++++++++++----- .../db/dao/ddp/instance/DDPInstanceDao.java | 29 +++++++++++++++++++ .../dsm/route/KitExpressRoute.java | 8 ++--- .../org/broadinstitute/dsm/util/KitUtil.java | 7 ++++- 4 files changed, 58 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java index ac044eba5..0d50a187a 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java @@ -167,12 +167,13 @@ public class KitRequestShipping extends KitRequest { private String trackingNumberTo; @ColumnName (DBConstants.DSM_TRACKING_RETURN) - private String trackingNumberReturn; + private String trackingReturnId; @ColumnName (DBConstants.TRACKING_ID) private String scannedTrackingNumber; private String trackingUrlTo; - private String trackingUrlReturn; + @ColumnName(DBConstants.DSM_TRACKING_URL_RETURN) + private String easypostTrackingReturnUrl; private String collaboratorParticipantId; @ColumnName (DBConstants.BSP_COLLABORATOR_PARTICIPANT_ID) @@ -274,7 +275,7 @@ public KitRequestShipping(Long dsmKitRequestId, Long dsmKitId, String easypostTo // shippingId = ddp_label !!! public KitRequestShipping(String participantId, String collaboratorParticipantId, String bspCollaboratorSampleId, String shippingId, String realm, String kitTypeName, long dsmKitRequestId, long dsmKitId, String labelUrlTo, String labelUrlReturn, - String trackingNumberTo, String trackingNumberReturn, + String trackingNumberTo, String trackingReturnId, String trackingUrlTo, String trackingUrlReturn, long scanDate, boolean error, String message, long receiveDate, String easypostAddressId, long deactivatedDate, String deactivationReason, String kitLabel, boolean express, String easypostToId, long labelDate, String easypostShipmentStatus, @@ -290,9 +291,9 @@ public KitRequestShipping(String participantId, String collaboratorParticipantId this.labelUrlTo = labelUrlTo; this.labelUrlReturn = labelUrlReturn; this.trackingNumberTo = trackingNumberTo; - this.trackingNumberReturn = trackingNumberReturn; + this.trackingReturnId = trackingReturnId; this.trackingUrlTo = trackingUrlTo; - this.trackingUrlReturn = trackingUrlReturn; + this.easypostTrackingReturnUrl = trackingUrlReturn; this.scanDate = scanDate; this.error = error; this.message = message; @@ -367,7 +368,7 @@ public static KitRequestShipping getKitRequestShipping(@NonNull ResultSet rs) th kitRequestShipping.setUpsTrackingStatus(rs.getString(DBConstants.UPS_STATUS_DESCRIPTION)); } - else if (StringUtils.isNotBlank(upsPackageTrackingNumber) && upsPackageTrackingNumber.equals(kitRequestShipping.getTrackingNumberReturn())) { + else if (StringUtils.isNotBlank(upsPackageTrackingNumber) && upsPackageTrackingNumber.equals(kitRequestShipping.getTrackingReturnId())) { kitRequestShipping.setUpsReturnStatus(rs.getString(DBConstants.UPS_STATUS_DESCRIPTION)); } @@ -993,7 +994,7 @@ public static void updateKit(long dsmKitId, String userId, DDPInstanceDto ddpIns // update kit with label information public static void updateKit(String dsmKitId, Shipment participantShipment, Shipment returnShipment, - String errorMessage, Address toAddress, boolean isExpress) { + String errorMessage, Address toAddress, boolean isExpress, DDPInstanceDto ddpInstanceDto) { KitRequestShipping kitRequestShipping = new KitRequestShipping(); SimpleResult results = inTransaction((conn) -> { SimpleResult dbVals = new SimpleResult(); @@ -1023,7 +1024,9 @@ public static void updateKit(String dsmKitId, Shipment participantShipment, Ship stmt.setString(4, returnShipment.getId()); stmt.setString(6, returnShipment.getTrackingCode()); stmt.setString(8, returnTracker.getPublicUrl()); - kitRequestShipping.setLabelUrlReturn(); + kitRequestShipping.setLabelUrlReturn(returnLabel.getLabelUrl()); + kitRequestShipping.setTrackingReturnId(returnShipment.getTrackingCode()); + kitRequestShipping.setEasypostTrackingReturnUrl(returnTracker.getPublicUrl()); } else { stmt.setString(2, null); @@ -1035,26 +1038,33 @@ public static void updateKit(String dsmKitId, Shipment participantShipment, Ship if (StringUtils.isNotBlank(errorMessage)) { stmt.setInt(9, 1); stmt.setString(10, errorMessage); + kitRequestShipping.setError(true); + kitRequestShipping.setMessage(errorMessage); logger.info("Added kit request with error message " + errorMessage); } else { stmt.setInt(9, 0); stmt.setString(10, null); + kitRequestShipping.setError(false); } if (toAddress != null) { stmt.setString(11, toAddress.getId()); + kitRequestShipping.setEasypostAddressId(toAddress.getId()); } else { stmt.setString(11, null); } if (isExpress) { stmt.setInt(12, 1); + kitRequestShipping.setExpress(true); } else { stmt.setInt(12, 0); + kitRequestShipping.setExpress(false); } stmt.setString(13, dsmKitId); + kitRequestShipping.setDsmKitId(Long.valueOf(dsmKitId)); int result = stmt.executeUpdate(); if (result != 1) { throw new RuntimeException("Error updating kit " + dsmKitId + " it was updating " + result + " rows"); @@ -1071,6 +1081,7 @@ public static void updateKit(String dsmKitId, Shipment participantShipment, Ship } else { logger.info("Updated kit w/ dsm_kit_id " + dsmKitId, results.resultException); + exportToES(kitRequestShipping, ddpInstanceDto, "dsmKitId", "dsmKitId", dsmKitId); } } diff --git a/src/main/java/org/broadinstitute/dsm/db/dao/ddp/instance/DDPInstanceDao.java b/src/main/java/org/broadinstitute/dsm/db/dao/ddp/instance/DDPInstanceDao.java index 297b092a7..8585212e5 100644 --- a/src/main/java/org/broadinstitute/dsm/db/dao/ddp/instance/DDPInstanceDao.java +++ b/src/main/java/org/broadinstitute/dsm/db/dao/ddp/instance/DDPInstanceDao.java @@ -93,6 +93,9 @@ public class DDPInstanceDao implements Dao { private static final String SQL_SELECT_DDP_INSTANCE_BY_INSTANCE_NAME = SQL_BASE_SELECT + "WHERE instance_name = ? "; + private static final String SQL_SELECT_DDP_INSTANCE_BY_INSTANCE_ID = SQL_BASE_SELECT + + "WHERE ddp_instance_id = ? "; + public static final String DDP_INSTANCE_ID = "ddp_instance_id"; @@ -390,4 +393,30 @@ public Optional getCollaboratorIdPrefixByStudyGuid(String studyGuid) { } return Optional.ofNullable((String) results.resultValue); } + + public Optional getDDPInstanceByInstanceId(Integer ddpInstanceId) { + SimpleResult results = inTransaction((conn) -> { + SimpleResult dbVals = new SimpleResult(); + try (PreparedStatement stmt = conn.prepareStatement(SQL_SELECT_DDP_INSTANCE_BY_INSTANCE_ID)) { + stmt.setInt(1, ddpInstanceId); + try (ResultSet rs = stmt.executeQuery()) { + if (rs.next()) { + dbVals.resultValue = getDdpInstanceDtoFromResultSet(rs); + } + } + catch (SQLException e) { + throw new RuntimeException("Error getting ddp instance for " + ddpInstanceId, e); + } + } + catch (SQLException ex) { + dbVals.resultException = ex; + } + return dbVals; + }); + + if (results.resultException != null) { + throw new RuntimeException("Couldn't get ddp instance for " + ddpInstanceId, results.resultException); + } + return Optional.ofNullable((DDPInstanceDto) results.resultValue); + } } diff --git a/src/main/java/org/broadinstitute/dsm/route/KitExpressRoute.java b/src/main/java/org/broadinstitute/dsm/route/KitExpressRoute.java index 9c6037c32..72d2d7464 100644 --- a/src/main/java/org/broadinstitute/dsm/route/KitExpressRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/KitExpressRoute.java @@ -98,20 +98,20 @@ public void expressKitRequest(@NonNull String kitRequestId, @NonNull String user } private void createExpressLabelToParticipant(@NonNull EasyPostUtil easyPostUtil, @NonNull KitRequestSettings kitRequestSettings, - @NonNull KitType kitType, @NonNull String kitId, @NonNull String addressToId, DDPInstanceDto ddpInstance) { + @NonNull KitType kitType, @NonNull String kitId, @NonNull String addressToId, DDPInstanceDto ddpInstanceDto) { String errorMessage = ""; Shipment participantShipment = null; Address toAddress = null; try { toAddress = KitRequestShipping.getToAddressId(easyPostUtil, kitRequestSettings, addressToId, null); - participantShipment = KitRequestShipping.getShipment(easyPostUtil, ddpInstance.getBillingReference(), + participantShipment = KitRequestShipping.getShipment(easyPostUtil, ddpInstanceDto.getBillingReference(), kitType, kitRequestSettings, toAddress, "FedEx", kitRequestSettings.getCarrierToId(), "FIRST_OVERNIGHT"); - doNotification(ddpInstance.getInstanceName()); + doNotification(ddpInstanceDto.getInstanceName()); } catch (Exception e) { errorMessage = "To: " + e.getMessage(); } - KitRequestShipping.updateKit(kitId, participantShipment, null, errorMessage, toAddress, true); + KitRequestShipping.updateKit(kitId, participantShipment, null, errorMessage, toAddress, true, ddpInstanceDto); } private String getKitId(@NonNull String kitRequestId) { diff --git a/src/main/java/org/broadinstitute/dsm/util/KitUtil.java b/src/main/java/org/broadinstitute/dsm/util/KitUtil.java index 899a893fa..cf3854923 100644 --- a/src/main/java/org/broadinstitute/dsm/util/KitUtil.java +++ b/src/main/java/org/broadinstitute/dsm/util/KitUtil.java @@ -280,7 +280,9 @@ private static void buyShipmentForKit(@NonNull EasyPostUtil easyPostUtil, @NonNu catch (Exception e) { errorMessage += "Return: " + e.getMessage(); } - KitRequestShipping.updateKit(dsmKitId, participantShipment, returnShipment, errorMessage, toAddress, false); + DDPInstanceDao ddpInstanceDao = new DDPInstanceDao(); + DDPInstanceDto ddpInstanceDto = ddpInstanceDao.getDDPInstanceByInstanceId(kitRequestSettings.getDdpInstanceId()).orElseThrow(); + KitRequestShipping.updateKit(dsmKitId, participantShipment, returnShipment, errorMessage, toAddress, false, ddpInstanceDto); } public static String getKitCollaboratorId(@NonNull String ddpParticipantId, @NonNull String realmId) { @@ -499,6 +501,9 @@ private static SimpleResult updateKit(@NonNull Connection conn, String status, L catch (Exception e) { dbVals.resultException = e; } + + exportToEs(); + return dbVals; } From aab3438fc37bb546c052adc6986a08318131f58a Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 25 Jan 2022 10:50:12 +0400 Subject: [PATCH 341/405] DDP-7443 | start change of KitStatusChangeRoute for es export --- .../dsm/db/KitRequestShipping.java | 17 ++++--- .../dsm/model/at/ReceiveKitRequest.java | 1 + .../dsm/route/KitStatusChangeRoute.java | 49 +++++++++++++++---- .../org/broadinstitute/dsm/util/KitUtil.java | 15 ++++-- .../dsm/KitStatusChangeRouteTest.java | 13 +++-- 5 files changed, 69 insertions(+), 26 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java index 0d50a187a..69e5c4ebb 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java @@ -170,10 +170,14 @@ public class KitRequestShipping extends KitRequest { private String trackingReturnId; @ColumnName (DBConstants.TRACKING_ID) - private String scannedTrackingNumber; - private String trackingUrlTo; + private String trackingId; + + @ColumnName(DBConstants.DSM_TRACKING_URL_TO) + private String easypostTrackingToUrl; + @ColumnName(DBConstants.DSM_TRACKING_URL_RETURN) private String easypostTrackingReturnUrl; + private String collaboratorParticipantId; @ColumnName (DBConstants.BSP_COLLABORATOR_PARTICIPANT_ID) @@ -276,7 +280,7 @@ public KitRequestShipping(Long dsmKitRequestId, Long dsmKitId, String easypostTo public KitRequestShipping(String participantId, String collaboratorParticipantId, String bspCollaboratorSampleId, String shippingId, String realm, String kitTypeName, long dsmKitRequestId, long dsmKitId, String labelUrlTo, String labelUrlReturn, String trackingNumberTo, String trackingReturnId, - String trackingUrlTo, String trackingUrlReturn, long scanDate, boolean error, String message, + String easypostTrackingToUrl, String trackingUrlReturn, long scanDate, boolean error, String message, long receiveDate, String easypostAddressId, long deactivatedDate, String deactivationReason, String kitLabel, boolean express, String easypostToId, long labelDate, String easypostShipmentStatus, String externalOrderNumber, boolean noReturn, String externalOrderStatus, String createdBy, String testResult, @@ -292,7 +296,7 @@ public KitRequestShipping(String participantId, String collaboratorParticipantId this.labelUrlReturn = labelUrlReturn; this.trackingNumberTo = trackingNumberTo; this.trackingReturnId = trackingReturnId; - this.trackingUrlTo = trackingUrlTo; + this.easypostTrackingToUrl = easypostTrackingToUrl; this.easypostTrackingReturnUrl = trackingUrlReturn; this.scanDate = scanDate; this.error = error; @@ -1009,7 +1013,7 @@ public static void updateKit(String dsmKitId, Shipment participantShipment, Ship kitRequestShipping.setLabelUrlTo(participantLabel.getLabelUrl()); kitRequestShipping.setEasypostToId(participantShipment.getId()); kitRequestShipping.setTrackingNumberTo(participantShipment.getTrackingCode()); - kitRequestShipping.setTrackingUrlTo(participantTracker.getPublicUrl()); //TODO + kitRequestShipping.setEasypostTrackingToUrl(participantTracker.getPublicUrl()); //TODO } else { stmt.setString(1, null); @@ -1427,7 +1431,8 @@ public static void reactivateKitRequest(@NonNull String dsmKitRequestId, String } } - private static void exportToES(KitRequestShipping kitRequestShipping, DDPInstanceDto ddpInstanceDto, String uniqueIdentifier, String fieldName, Object fieldValue) { + public static void exportToES(KitRequestShipping kitRequestShipping, DDPInstanceDto ddpInstanceDto, String uniqueIdentifier, + String fieldName, Object fieldValue) { Generator paramsGenerator = new ParamsGenerator(kitRequestShipping, ddpInstanceDto.getInstanceName()); ScriptBuilder scriptBuilder = new NestedScriptBuilder(paramsGenerator.getPropertyName(), uniqueIdentifier); MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder(fieldName, fieldValue); diff --git a/src/main/java/org/broadinstitute/dsm/model/at/ReceiveKitRequest.java b/src/main/java/org/broadinstitute/dsm/model/at/ReceiveKitRequest.java index 3a701fd83..76356e5c5 100644 --- a/src/main/java/org/broadinstitute/dsm/model/at/ReceiveKitRequest.java +++ b/src/main/java/org/broadinstitute/dsm/model/at/ReceiveKitRequest.java @@ -59,6 +59,7 @@ public static boolean receiveATKitRequest(@NonNull NotificationUtil notification } private static boolean updateData(@NonNull String data, @NonNull long participantDataId) { + //TODO -> update participantData ES SimpleResult results = inTransaction((conn) -> { SimpleResult dbVals = new SimpleResult(); try (PreparedStatement stmt = conn.prepareStatement(SQL_UPDATE_KIT_REQUEST)){ diff --git a/src/main/java/org/broadinstitute/dsm/route/KitStatusChangeRoute.java b/src/main/java/org/broadinstitute/dsm/route/KitStatusChangeRoute.java index 7bc5508cf..decd11ca4 100644 --- a/src/main/java/org/broadinstitute/dsm/route/KitStatusChangeRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/KitStatusChangeRoute.java @@ -8,7 +8,10 @@ import org.broadinstitute.ddp.db.TransactionWrapper; import org.broadinstitute.ddp.handlers.util.Result; import org.broadinstitute.dsm.db.DDPInstance; +import org.broadinstitute.dsm.db.KitRequestShipping; +import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; import org.broadinstitute.dsm.db.dao.ddp.kitrequest.KitRequestDao; +import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; import org.broadinstitute.dsm.db.dto.ddp.kitrequest.KitRequestDto; import org.broadinstitute.dsm.model.KitDDPNotification; import org.broadinstitute.dsm.model.at.ReceiveKitRequest; @@ -17,6 +20,7 @@ import org.broadinstitute.dsm.util.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import spark.QueryParamsMap; import spark.Request; import spark.Response; @@ -27,6 +31,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import static org.broadinstitute.ddp.db.TransactionWrapper.inTransaction; @@ -44,6 +49,9 @@ public KitStatusChangeRoute(@NonNull NotificationUtil notificationUtil) { public Object processRequest(Request request, Response response, String userId) throws Exception { String requestBody = request.body(); String userIdRequest = UserUtil.getUserId(request); + QueryParamsMap queryParams = request.queryMap(); + String realm = queryParams.get(RoutePath.REALM).value(); + DDPInstanceDto ddpInstanceDto = new DDPInstanceDao().getDDPInstanceByInstanceName(realm).orElseThrow(); if (UserUtil.checkUserAccess(null, userId, "kit_shipping", userIdRequest) || UserUtil.checkUserAccess(null, userId, "kit_receiving", userIdRequest)) { List scanErrorList = new ArrayList<>(); @@ -52,16 +60,16 @@ public Object processRequest(Request request, Response response, String userId) int labelCount = scans.size(); if (labelCount > 0) { if (request.url().endsWith(RoutePath.FINAL_SCAN_REQUEST)) { - updateKits(RoutePath.FINAL_SCAN_REQUEST, scans, currentTime, scanErrorList, userIdRequest); + updateKits(RoutePath.FINAL_SCAN_REQUEST, scans, currentTime, scanErrorList, userIdRequest, ddpInstanceDto); } else if (request.url().endsWith(RoutePath.TRACKING_SCAN_REQUEST)) { - updateKits(RoutePath.TRACKING_SCAN_REQUEST, scans, currentTime, scanErrorList, userIdRequest); + updateKits(RoutePath.TRACKING_SCAN_REQUEST, scans, currentTime, scanErrorList, userIdRequest, ddpInstanceDto); } else if (request.url().endsWith(RoutePath.SENT_KIT_REQUEST)) { - updateKits(RoutePath.SENT_KIT_REQUEST, scans, currentTime, scanErrorList, userIdRequest); + updateKits(RoutePath.SENT_KIT_REQUEST, scans, currentTime, scanErrorList, userIdRequest, ddpInstanceDto); } else if (request.url().endsWith(RoutePath.RECEIVED_KIT_REQUEST)) { - updateKits(RoutePath.RECEIVED_KIT_REQUEST, scans, currentTime, scanErrorList, userIdRequest); + updateKits(RoutePath.RECEIVED_KIT_REQUEST, scans, currentTime, scanErrorList, userIdRequest, ddpInstanceDto); } else { logger.error("Endpoint was not known " + request.url()); @@ -76,7 +84,8 @@ else if (request.url().endsWith(RoutePath.RECEIVED_KIT_REQUEST)) { } } - public void updateKits(@NonNull String changeType, @NonNull JsonArray scans, @NonNull long currentTime, @NonNull List scanErrorList, @NonNull String userId) { + public void updateKits(@NonNull String changeType, @NonNull JsonArray scans, long currentTime, @NonNull List scanErrorList, + @NonNull String userId, DDPInstanceDto ddpInstanceDto) { for (JsonElement scan : scans) { String addValue = null; String kit = null; @@ -87,7 +96,7 @@ public void updateKits(@NonNull String changeType, @NonNull JsonArray scans, @No if (checkKitLabel(TransactionWrapper.getSqlFromConfig(ApplicationConfigConstants.GET_KIT_TYPE_NEED_TRACKING_BY_DDP_LABEL), kit)) { //check if kit_label is in tracking table if (checkKitLabel(TransactionWrapper.getSqlFromConfig(ApplicationConfigConstants.GET_FOUND_IF_KIT_LABEL_ALREADY_EXISTS_IN_TRACKING_TABLE), addValue)) { - updateKit(changeType, kit, addValue, currentTime, scanErrorList, userId); + updateKit(changeType, kit, addValue, currentTime, scanErrorList, userId, ddpInstanceDto); KitRequestDao kitRequestDao = new KitRequestDao(); KitRequestDto kitRequestByLabel = kitRequestDao.getKitRequestByLabel(kit); if (kitRequestByLabel != null) { @@ -99,17 +108,17 @@ public void updateKits(@NonNull String changeType, @NonNull JsonArray scans, @No } } else { - updateKit(changeType, kit, addValue, currentTime, scanErrorList, userId); + updateKit(changeType, kit, addValue, currentTime, scanErrorList, userId, ddpInstanceDto); } } else if (RoutePath.TRACKING_SCAN_REQUEST.equals(changeType)) { addValue = scan.getAsJsonObject().get("leftValue").getAsString(); kit = scan.getAsJsonObject().get("rightValue").getAsString(); - updateKit(changeType, kit, addValue, currentTime, scanErrorList, userId); + updateKit(changeType, kit, addValue, currentTime, scanErrorList, userId, ddpInstanceDto); } else if (RoutePath.SENT_KIT_REQUEST.equals(changeType) || RoutePath.RECEIVED_KIT_REQUEST.equals(changeType)) { kit = scan.getAsJsonObject().get("kit").getAsString(); - updateKit(changeType, kit, addValue, currentTime, scanErrorList, userId); + updateKit(changeType, kit, addValue, currentTime, scanErrorList, userId, ddpInstanceDto); } else { throw new RuntimeException("Endpoint was not known"); @@ -133,15 +142,24 @@ private static void writeSampleSentToES(KitRequestDto kitRequest) { } } - private void updateKit(@NonNull String changeType, @NonNull String kit, String addValue, @NonNull long currentTime, @NonNull List scanErrorList, @NonNull String userId) { + private void updateKit(@NonNull String changeType, @NonNull String kit, String addValue, long currentTime, + @NonNull List scanErrorList, @NonNull String userId, + DDPInstanceDto ddpInstanceDto) { + KitRequestShipping kitRequestShipping = new KitRequestShipping(); SimpleResult results = inTransaction((conn) -> { SimpleResult dbVals = new SimpleResult(); String query = null; if (RoutePath.FINAL_SCAN_REQUEST.equals(changeType) || RoutePath.SENT_KIT_REQUEST.equals(changeType)) { query = TransactionWrapper.getSqlFromConfig(ApplicationConfigConstants.UPDATE_KIT_REQUEST); + kitRequestShipping.setScanDate(currentTime); + kitRequestShipping.setKitLabel(addValue); } else if (RoutePath.TRACKING_SCAN_REQUEST.equals(changeType)) { query = TransactionWrapper.getSqlFromConfig(ApplicationConfigConstants.INSERT_KIT_TRACKING); + //add value is value for tracking_id in ddp_kit_tracking and is considered as trackingReturnId in KitRequestShipping + kitRequestShipping.setScanDate(currentTime); + kitRequestShipping.setTrackingReturnId(addValue); + kitRequestShipping.setKitLabel(kit); } else if (RoutePath.RECEIVED_KIT_REQUEST.equals(changeType)) { query = KitUtil.SQL_UPDATE_KIT_RECEIVED; @@ -218,6 +236,17 @@ else if (RoutePath.RECEIVED_KIT_REQUEST.equals(changeType)) { else { logger.error("Error something went wrong at the scan pages"); } + } else { + if (Objects.isNull(ddpInstanceDto)) return; + if (RoutePath.FINAL_SCAN_REQUEST.equals(changeType) || RoutePath.SENT_KIT_REQUEST.equals(changeType)) { + KitRequestShipping.exportToES(kitRequestShipping, ddpInstanceDto, "ddpLabel", "ddpLabel", kit); + } + else if (RoutePath.TRACKING_SCAN_REQUEST.equals(changeType)) { + KitRequestShipping.exportToES(kitRequestShipping, ddpInstanceDto, "kitLabel", "kitLable", kit); + } + else if (RoutePath.RECEIVED_KIT_REQUEST.equals(changeType)) { + //TODO - sm_id + } } } diff --git a/src/main/java/org/broadinstitute/dsm/util/KitUtil.java b/src/main/java/org/broadinstitute/dsm/util/KitUtil.java index cf3854923..33c46f9d7 100644 --- a/src/main/java/org/broadinstitute/dsm/util/KitUtil.java +++ b/src/main/java/org/broadinstitute/dsm/util/KitUtil.java @@ -429,7 +429,12 @@ else if (status.equals(detail.getStatus())) { kitMessage = message; } } - SimpleResult result = updateKit(conn, status, deliveredDate, kitMessage, kitRequest.getDsmKitId()); + DDPInstanceDto ddpInstanceDto = new DDPInstanceDto.Builder() + .withInstanceName(ddpInstance.getName()) + .withEsParticipantIndex(ddpInstance.getParticipantIndexES()) + .build(); + SimpleResult result = updateKit(conn, status, deliveredDate, kitMessage, + kitRequest.getDsmKitId(), ddpInstanceDto); if (result.resultException != null) { logger.error("Error updating kit for kit request " + kitRequest.getDsmKitRequestId() + " of realm " + ddpInstance.getName(), result.resultException); } @@ -485,7 +490,7 @@ private static List getKitRequestsToCheckStatus(@NonNull Str // update kit with label trigger user and date private static SimpleResult updateKit(@NonNull Connection conn, String status, Long date, String message, - long dsmKitId) { + long dsmKitId, DDPInstanceDto ddpInstanceDto) { SimpleResult dbVals = new SimpleResult(); try (PreparedStatement stmt = conn.prepareStatement(SQL_UPDATE_KIT)) { stmt.setString(1, status); @@ -502,7 +507,11 @@ private static SimpleResult updateKit(@NonNull Connection conn, String status, L dbVals.resultException = e; } - exportToEs(); + KitRequestShipping kitRequestShipping = new KitRequestShipping(); + kitRequestShipping.setEasypostShipmentStatus(status); + kitRequestShipping.setMessage(message); + kitRequestShipping.setDsmKitId(dsmKitId); + KitRequestShipping.exportToES(kitRequestShipping, ddpInstanceDto, "dsmKitId", "dsmKitId", dsmKitId); return dbVals; } diff --git a/src/test/java/org/broadinstitute/dsm/KitStatusChangeRouteTest.java b/src/test/java/org/broadinstitute/dsm/KitStatusChangeRouteTest.java index 4aac4808d..6d91f0966 100644 --- a/src/test/java/org/broadinstitute/dsm/KitStatusChangeRouteTest.java +++ b/src/test/java/org/broadinstitute/dsm/KitStatusChangeRouteTest.java @@ -3,7 +3,6 @@ import com.google.gson.JsonArray; import com.google.gson.JsonParser; import org.broadinstitute.ddp.db.TransactionWrapper; -import org.broadinstitute.dsm.TestHelper; import org.broadinstitute.dsm.route.KitStatusChangeRoute; import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.util.DBTestUtil; @@ -54,7 +53,7 @@ public void getBSPKitQuery() throws Exception { String json = TestUtil.readFile("FinalScanPayload.json"); JsonArray scans = (JsonArray) (new JsonParser().parse(json)); List scanErrorList = new ArrayList<>(); - route.updateKits("finalScan", scans, System.currentTimeMillis(), scanErrorList, "3"); + route.updateKits("finalScan", scans, System.currentTimeMillis(), scanErrorList, "3", null); List strings = new ArrayList<>(); strings.add(FAKE_LATEST_KIT + "_1"); @@ -103,7 +102,7 @@ public void trackingScan() throws Exception { String json = TestUtil.readFile("TrackingScanPayload.json"); JsonArray scans = (JsonArray) (new JsonParser().parse(json)); List scanErrorList = new ArrayList<>(); - route.updateKits("trackingScan", scans, System.currentTimeMillis(), scanErrorList, "3"); + route.updateKits("trackingScan", scans, System.currentTimeMillis(), scanErrorList, "3", null); List strings = new ArrayList<>(); strings.add(FAKE_DSM_LABEL_UID + "_3"); @@ -111,7 +110,7 @@ public void trackingScan() throws Exception { //check if kitlabel was added to tracking table Assert.assertEquals("1", count); - route.updateKits("finalScan", scans, System.currentTimeMillis(), scanErrorList, "3"); + route.updateKits("finalScan", scans, System.currentTimeMillis(), scanErrorList, "3", null); strings = new ArrayList<>(); strings.add(FAKE_LATEST_KIT + "_3"); @@ -129,7 +128,7 @@ public void noTrackingScan() throws Exception { String json = TestUtil.readFile("TrackingScanPayload.json"); JsonArray scans = (JsonArray) (new JsonParser().parse(json)); List scanErrorList = new ArrayList<>(); - route.updateKits("finalScan", scans, System.currentTimeMillis(), scanErrorList, "3"); + route.updateKits("finalScan", scans, System.currentTimeMillis(), scanErrorList, "3", null); List strings = new ArrayList<>(); strings.add(FAKE_LATEST_KIT + "_4"); @@ -152,7 +151,7 @@ public void doubleTrackingScan() throws Exception { String json = TestUtil.readFile("TrackingScanPayload.json"); JsonArray scans = (JsonArray) (new JsonParser().parse(json)); List scanErrorList = new ArrayList<>(); - route.updateKits("trackingScan", scans, System.currentTimeMillis(), scanErrorList, "3"); + route.updateKits("trackingScan", scans, System.currentTimeMillis(), scanErrorList, "3", null); List strings = new ArrayList<>(); strings.add(FAKE_DSM_LABEL_UID + "_3"); @@ -161,6 +160,6 @@ public void doubleTrackingScan() throws Exception { Assert.assertEquals("1", count); scanErrorList = new ArrayList<>(); - route.updateKits("trackingScan", scans, System.currentTimeMillis(), scanErrorList, "3"); + route.updateKits("trackingScan", scans, System.currentTimeMillis(), scanErrorList, "3", null); } } From 9982ee793896453cc2f87ae1dc8a0fda7a0accf4 Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 25 Jan 2022 12:14:59 +0400 Subject: [PATCH 342/405] DDP-7443 | add export to BSPKit, change representation of notification recipients from String to List --- .../dsm/db/KitRequestShipping.java | 4 ++-- .../db/dao/ddp/instance/DDPInstanceDao.java | 6 ++++- .../db/dto/ddp/instance/DDPInstanceDto.java | 23 +++++++++++++------ .../broadinstitute/dsm/export/ExportToES.java | 2 ++ .../export/WorkflowAndFamilyIdExporter.java | 6 +---- .../org/broadinstitute/dsm/model/BSPKit.java | 21 ++++++++++++++--- .../dsm/model/at/ReceiveKitRequest.java | 23 +++++++++++++------ .../dsm/route/KitStatusChangeRoute.java | 3 +++ 8 files changed, 63 insertions(+), 25 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java index 69e5c4ebb..f05a76cd9 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java @@ -1431,9 +1431,9 @@ public static void reactivateKitRequest(@NonNull String dsmKitRequestId, String } } - public static void exportToES(KitRequestShipping kitRequestShipping, DDPInstanceDto ddpInstanceDto, String uniqueIdentifier, + public static void exportToES(Object source, DDPInstanceDto ddpInstanceDto, String uniqueIdentifier, String fieldName, Object fieldValue) { - Generator paramsGenerator = new ParamsGenerator(kitRequestShipping, ddpInstanceDto.getInstanceName()); + Generator paramsGenerator = new ParamsGenerator(source, ddpInstanceDto.getInstanceName()); ScriptBuilder scriptBuilder = new NestedScriptBuilder(paramsGenerator.getPropertyName(), uniqueIdentifier); MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder(fieldName, fieldValue); NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("dsm.kitRequestShipping", matchQueryBuilder, ScoreMode.Avg); diff --git a/src/main/java/org/broadinstitute/dsm/db/dao/ddp/instance/DDPInstanceDao.java b/src/main/java/org/broadinstitute/dsm/db/dao/ddp/instance/DDPInstanceDao.java index 8585212e5..a2629c6d0 100644 --- a/src/main/java/org/broadinstitute/dsm/db/dao/ddp/instance/DDPInstanceDao.java +++ b/src/main/java/org/broadinstitute/dsm/db/dao/ddp/instance/DDPInstanceDao.java @@ -165,7 +165,7 @@ public int create(DDPInstanceDto ddpInstanceDto) { stmt.setObject(11, ddpInstanceDto.getMrAttentionFlagD()); stmt.setObject(12, ddpInstanceDto.getTissueAttentionFlagD()); stmt.setBoolean(13, ddpInstanceDto.getAuth0Token()); - stmt.setString(14, ddpInstanceDto.getNotificationRecipients()); + stmt.setString(14, getNotificationsAsSequence(ddpInstanceDto)); stmt.setBoolean(15, ddpInstanceDto.getMigratedDdp()); stmt.setString(16, ddpInstanceDto.getBillingReference()); stmt.setString(17, ddpInstanceDto.getEsParticipantIndex()); @@ -188,6 +188,10 @@ public int create(DDPInstanceDto ddpInstanceDto) { return (int) simpleResult.resultValue; } + private String getNotificationsAsSequence(DDPInstanceDto ddpInstanceDto) { + return ddpInstanceDto.getNotificationRecipients().toString().replace("[", "").replace("]", "").replace("\\s", ""); + } + @Override public int delete(int id) { SimpleResult simpleResult = inTransaction(conn -> { diff --git a/src/main/java/org/broadinstitute/dsm/db/dto/ddp/instance/DDPInstanceDto.java b/src/main/java/org/broadinstitute/dsm/db/dto/ddp/instance/DDPInstanceDto.java index f265d22aa..7bdcd85fa 100644 --- a/src/main/java/org/broadinstitute/dsm/db/dto/ddp/instance/DDPInstanceDto.java +++ b/src/main/java/org/broadinstitute/dsm/db/dto/ddp/instance/DDPInstanceDto.java @@ -1,14 +1,13 @@ package org.broadinstitute.dsm.db.dto.ddp.instance; +import lombok.Data; +import org.apache.commons.lang3.StringUtils; + import java.lang.reflect.Field; -import java.util.Arrays; -import java.util.Map; +import java.util.*; import java.util.function.Function; import java.util.stream.Collectors; -import lombok.Data; -import lombok.NonNull; - @Data public class DDPInstanceDto { @@ -33,12 +32,21 @@ public class DDPInstanceDto { String esActivityDefinitionIndex; String esUsersIndex; + public List getNotificationRecipients() { + if (StringUtils.isBlank(notificationRecipients)) { + return Collections.emptyList(); + } + notificationRecipients = notificationRecipients.replaceAll("\\s", ""); + return Arrays.asList(notificationRecipients.split(",")); + } + + private DDPInstanceDto(Builder builder) { Class ddpInstanceDtoClazz = DDPInstanceDto.class; Field[] declaredFields = ddpInstanceDtoClazz.getDeclaredFields(); Map builderFieldsMap = Arrays.stream(builder.getClass().getDeclaredFields()).collect(Collectors.toMap(Field::getName, Function.identity())); - for (Field field: declaredFields) { + for (Field field : declaredFields) { try { field.set(this, builderFieldsMap.get(field.getName()).get(builder)); } catch (IllegalAccessException e) { @@ -69,7 +77,8 @@ public static class Builder { String esActivityDefinitionIndex; String esUsersIndex; - public Builder() {} + public Builder() { + } public Builder withDdpInstanceId(int ddpInstanceId) { this.ddpInstanceId = ddpInstanceId; diff --git a/src/main/java/org/broadinstitute/dsm/export/ExportToES.java b/src/main/java/org/broadinstitute/dsm/export/ExportToES.java index a1f2d53b3..d7472cda5 100644 --- a/src/main/java/org/broadinstitute/dsm/export/ExportToES.java +++ b/src/main/java/org/broadinstitute/dsm/export/ExportToES.java @@ -5,6 +5,8 @@ import com.google.gson.Gson; import org.broadinstitute.dsm.db.DDPInstance; +import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; +import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/org/broadinstitute/dsm/export/WorkflowAndFamilyIdExporter.java b/src/main/java/org/broadinstitute/dsm/export/WorkflowAndFamilyIdExporter.java index 5d39fc68d..cba6dc237 100644 --- a/src/main/java/org/broadinstitute/dsm/export/WorkflowAndFamilyIdExporter.java +++ b/src/main/java/org/broadinstitute/dsm/export/WorkflowAndFamilyIdExporter.java @@ -16,11 +16,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Deque; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; public class WorkflowAndFamilyIdExporter implements Exporter { diff --git a/src/main/java/org/broadinstitute/dsm/model/BSPKit.java b/src/main/java/org/broadinstitute/dsm/model/BSPKit.java index 0ee7fd59f..4f92e971e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/BSPKit.java +++ b/src/main/java/org/broadinstitute/dsm/model/BSPKit.java @@ -5,8 +5,11 @@ import org.broadinstitute.ddp.db.TransactionWrapper; import org.broadinstitute.dsm.db.DDPInstance; import org.broadinstitute.dsm.db.InstanceSettings; +import org.broadinstitute.dsm.db.KitRequestShipping; +import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; import org.broadinstitute.dsm.db.dao.ddp.kitrequest.KitRequestDao; import org.broadinstitute.dsm.db.dao.kit.BSPKitDao; +import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; import org.broadinstitute.dsm.db.dto.kit.BSPKitDto; import org.broadinstitute.dsm.db.dto.settings.InstanceSettingsDto; import org.broadinstitute.dsm.model.bsp.BSPKitInfo; @@ -79,7 +82,7 @@ public Optional receiveBSPKit(String kitLabel, NotificationUtil noti if (StringUtils.isBlank(maybeBspKitQueryResult.getDdpParticipantId())) { throw new RuntimeException("No participant id for " + kitLabel + " from " + maybeBspKitQueryResult.getInstanceName()); } - logger.info("participant id is " + maybeBspKitQueryResult.getDdpParticipantId()); + logger.info("particpant id is " + maybeBspKitQueryResult.getDdpParticipantId()); DDPInstance ddpInstance = DDPInstance.getDDPInstance(maybeBspKitQueryResult.getInstanceName()); InstanceSettings instanceSettings = new InstanceSettings(); InstanceSettingsDto instanceSettingsDto = instanceSettings.getInstanceSettings(maybeBspKitQueryResult.getInstanceName()); @@ -106,10 +109,10 @@ public Optional receiveBSPKit(String kitLabel, NotificationUtil noti logger.error("Instance settings behavior for kit was not known " + received.getType()); } } - bspKitDao.setKitReceivedAndTriggerDDP(kitLabel, triggerDDP, maybeBspKitQueryResult); + updateKitAndExport(kitLabel, bspKitDao, maybeBspKitQueryResult, triggerDDP); } }, () -> { - bspKitDao.setKitReceivedAndTriggerDDP(kitLabel, true, maybeBspKitQueryResult); + updateKitAndExport(kitLabel, bspKitDao, maybeBspKitQueryResult, true); }); String bspParticipantId = maybeBspKitQueryResult.getBspParticipantId(); @@ -138,6 +141,18 @@ public Optional receiveBSPKit(String kitLabel, NotificationUtil noti } + private void updateKitAndExport(String kitLabel, BSPKitDao bspKitDao, BSPKitDto maybeBspKitQueryResult, boolean triggerDDP) { + long receivedDate = System.currentTimeMillis(); + bspKitDao.setKitReceivedAndTriggerDDP(kitLabel, triggerDDP, maybeBspKitQueryResult); + + KitRequestShipping kitRequestShipping = new KitRequestShipping(); + kitRequestShipping.setReceiveDate(receivedDate); + kitRequestShipping.setKitLabel(kitLabel); + + DDPInstanceDto ddpInstanceDto = new DDPInstanceDao().getDDPInstanceByInstanceName(maybeBspKitQueryResult.getInstanceName()).orElseThrow(); + KitRequestShipping.exportToES(kitRequestShipping, ddpInstanceDto, "kitLabel", "kitLabel", kitLabel); + } + private void writeSampleReceivedToES(DDPInstance ddpInstance, BSPKitDto bspKitInfo) { String kitRequestId = new KitRequestDao().getKitRequestIdByBSPParticipantId(bspKitInfo.getBspParticipantId()); Map nameValuesMap = new HashMap<>(); diff --git a/src/main/java/org/broadinstitute/dsm/model/at/ReceiveKitRequest.java b/src/main/java/org/broadinstitute/dsm/model/at/ReceiveKitRequest.java index 76356e5c5..ad16ad8c4 100644 --- a/src/main/java/org/broadinstitute/dsm/model/at/ReceiveKitRequest.java +++ b/src/main/java/org/broadinstitute/dsm/model/at/ReceiveKitRequest.java @@ -5,8 +5,11 @@ import lombok.NonNull; import org.apache.commons.lang3.StringUtils; import org.broadinstitute.ddp.db.SimpleResult; -import org.broadinstitute.dsm.db.DDPInstance; +import org.broadinstitute.dsm.db.KitRequestShipping; import org.broadinstitute.dsm.db.ParticipantData; +import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; +import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; +import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; import org.broadinstitute.dsm.util.NotificationUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,11 +43,10 @@ public static boolean receiveATKitRequest(@NonNull NotificationUtil notification String formattedDate = formatter.format(now); data.put(RECEIVED_DATE, formattedDate); data.put(GENOME_STUDY_STATUS, "3"); - + DDPInstanceDto ddpInstanceDto = new DDPInstanceDao().getDDPInstanceByInstanceName("atcp").orElseThrow(); String dataString = new Gson().toJson(data); - if (updateData(dataString, participantData.getParticipantDataId())) { - DDPInstance ddpInstance = DDPInstance.getDDPInstance("atcp"); - List recipients = ddpInstance.getNotificationRecipient(); + if (updateData(dataString, participantData.getParticipantDataId(), ddpInstanceDto)) { + List recipients = ddpInstanceDto.getNotificationRecipients(); if (recipients != null && !recipients.isEmpty()) { for (String recipient : recipients) { String message = NOTIFICATION_MESSAGE.replace("GENOME_STUDY_SPIT_KIT_BARCODE", mfBarcode).replace("GENOME_STUDY_DATE_RECEIVED", formattedDate); @@ -58,8 +60,7 @@ public static boolean receiveATKitRequest(@NonNull NotificationUtil notification return false; } - private static boolean updateData(@NonNull String data, @NonNull long participantDataId) { - //TODO -> update participantData ES + private static boolean updateData(@NonNull String data, @NonNull long participantDataId, DDPInstanceDto ddpInstanceDto) { SimpleResult results = inTransaction((conn) -> { SimpleResult dbVals = new SimpleResult(); try (PreparedStatement stmt = conn.prepareStatement(SQL_UPDATE_KIT_REQUEST)){ @@ -82,6 +83,14 @@ private static boolean updateData(@NonNull String data, @NonNull long participan if (results.resultException != null) { throw new RuntimeException("Error setting AT kit to received with id " + participantDataId, results.resultException); } + + ParticipantDataDto participantDataDto = new ParticipantDataDto.Builder() + .withData(data) + .withParticipantDataId((int) participantDataId) + .build(); + + KitRequestShipping.exportToES(participantDataDto, ddpInstanceDto, "participantDataId", "participantDataId", participantDataId); + return true; } } diff --git a/src/main/java/org/broadinstitute/dsm/route/KitStatusChangeRoute.java b/src/main/java/org/broadinstitute/dsm/route/KitStatusChangeRoute.java index decd11ca4..f1d8fcdab 100644 --- a/src/main/java/org/broadinstitute/dsm/route/KitStatusChangeRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/KitStatusChangeRoute.java @@ -14,6 +14,7 @@ import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; import org.broadinstitute.dsm.db.dto.ddp.kitrequest.KitRequestDto; import org.broadinstitute.dsm.model.KitDDPNotification; +import org.broadinstitute.dsm.model.KitRequestSettings; import org.broadinstitute.dsm.model.at.ReceiveKitRequest; import org.broadinstitute.dsm.security.RequestHandler; import org.broadinstitute.dsm.statics.*; @@ -182,9 +183,11 @@ else if (RoutePath.RECEIVED_KIT_REQUEST.equals(changeType)) { if (kitDDPNotification != null) { EventUtil.triggerDDP(conn, kitDDPNotification); } + KitRequestShipping.exportToES(kitRequestShipping, ddpInstanceDto, "kitLabel", "kitLabel", addValue); } else if (RoutePath.TRACKING_SCAN_REQUEST.equals(changeType)) { logger.info("Added tracking for kit w/ kit_label " + kit); + KitRequestShipping.exportToES(kitRequestShipping, ddpInstanceDto, "kitLabel", "kitLabel", addValue); } else if (RoutePath.RECEIVED_KIT_REQUEST.equals(changeType)) { logger.info("Updated kitRequest w/ SM-ID kit_label " + kit); From cd86a21ed99b165a2785db9cd2266932caa4db1c Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 25 Jan 2022 17:49:11 +0400 Subject: [PATCH 343/405] DDP-7443 | fix ES NPE error --- .../dsm/db/KitRequestShipping.java | 27 +++++++++++-------- .../export/painless/NestedScriptBuilder.java | 5 +++- .../export/painless/ParamsGenerator.java | 5 ++-- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java index f05a76cd9..6132514ae 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java @@ -11,6 +11,7 @@ import org.broadinstitute.ddp.db.SimpleResult; import org.broadinstitute.ddp.db.TransactionWrapper; import org.broadinstitute.dsm.DSMServer; +import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; import org.broadinstitute.dsm.db.structure.ColumnName; import org.broadinstitute.dsm.db.structure.DbDateConversion; @@ -133,7 +134,6 @@ public class KitRequestShipping extends KitRequest { private static final String QUEUE = "queue"; - @ColumnName(DBConstants.ERROR) private static final String ERROR = "error"; private static final String SENT = "sent"; private static final String RECEIVED = "received"; @@ -896,16 +896,21 @@ public static String writeRequest(@NonNull String instanceId, @NonNull String dd } if (Objects.nonNull(ddpInstance)) { - KitRequestShipping kitRequestShipping = new KitRequestShipping(ddpParticipantId, collaboratorPatientId, collaboratorSampleId, null, - null, null, (long) results.resultValue, 0, null, null, null, null, null, null, 0, false, errorMessage, 0, null, - 0, null, null, false, null, 0, null, externalOrderNumber, false, null, createdBy, null, - null, null, 0, false, uploadReason, null, null, null); - - Generator paramsGenerator = new ParamsGenerator(kitRequestShipping, ddpInstance.getName()); - ScriptBuilder scriptBuilder = new NestedScriptBuilder(paramsGenerator.getPropertyName(), "dsmKitRequestId"); - MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("_id", Exportable.getParticipantGuid(ddpParticipantId, ddpInstance.getParticipantIndexES())); - UpsertPainless upsertPainless = new UpsertPainless(paramsGenerator, ddpInstance.getParticipantIndexES(), scriptBuilder, matchQueryBuilder); - upsertPainless.export(); + KitRequestShipping kitRequestShipping = new KitRequestShipping(); + kitRequestShipping.setParticipantId(ddpParticipantId); + kitRequestShipping.setCollaboratorParticipantId(collaboratorPatientId); + kitRequestShipping.setBspCollaboratorSampleId(collaboratorSampleId); + kitRequestShipping.setDsmKitRequestId(Long.parseLong(String.valueOf(results.resultValue))); + kitRequestShipping.setMessage(errorMessage); + kitRequestShipping.setExternalOrderNumber(externalOrderNumber); + kitRequestShipping.setCreatedBy(createdBy); + kitRequestShipping.setUploadReason(uploadReason); + + DDPInstanceDto ddpInstanceDto = + new DDPInstanceDao().getDDPInstanceByInstanceId(Integer.valueOf(ddpInstance.getDdpInstanceId())).orElseThrow(); + + exportToES(kitRequestShipping, ddpInstanceDto, "dsmKitRequestId", "_id", Exportable.getParticipantGuid(ddpParticipantId, + ddpInstance.getParticipantIndexES())); } logger.info("Added kitRequest w/ ddpKitRequestId " + ddpKitRequestId); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedScriptBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedScriptBuilder.java index 2699cf56a..54a7b1d9b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedScriptBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedScriptBuilder.java @@ -3,7 +3,10 @@ public class NestedScriptBuilder implements ScriptBuilder { private final static String SCRIPT = "if (ctx._source.dsm.#propertyName == null) {ctx._source.dsm.#propertyName = [params.dsm.#propertyName]} " + - "else {def targets = ctx._source.dsm.#propertyName.findAll(obj -> obj.#uniqueIdentifier == params.dsm.#propertyName.#uniqueIdentifier); " + + "else {def targets = ctx._source.dsm.#propertyName.findAll(obj -> obj.containsKey('#uniqueIdentifier') && obj.#uniqueIdentifier" + + " == params.dsm" + + ".#propertyName" + + ".#uniqueIdentifier); " + "if (targets.size() == 0) { ctx._source.dsm.#propertyName.add(params.dsm.#propertyName) } " + "else { for(target in targets) { for (entry in params.dsm.#propertyName.entrySet()) { target.put(entry.getKey(), entry.getValue()) } }}}"; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGenerator.java index 035a855e7..d0a99b081 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGenerator.java @@ -21,9 +21,8 @@ public ParamsGenerator(Object source, String realm) { @Override public Map generate() { Map fieldsMap = Util.transformObjectToMap(source, realm); - return new HashMap<>(Map.of(PARAMS, - new HashMap<>(Map.of(ESObjectConstants.DSM, - new HashMap<>(Map.of(getPropertyName(), fieldsMap)))))); + return new HashMap<>(Map.of(ESObjectConstants.DSM, + new HashMap<>(Map.of(getPropertyName(), fieldsMap)))); } @Override From 89a180a3b96bb3cf4c1c9641099335ba2bf6e061 Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 25 Jan 2022 18:54:59 +0400 Subject: [PATCH 344/405] DDP-7443 | fix bug related to parsing string to long --- .../java/org/broadinstitute/dsm/db/KitRequestShipping.java | 2 +- .../elastic/migration/DynamicFieldsMappingMigratorTest.java | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java index 6132514ae..7a17dcd1e 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java @@ -961,7 +961,7 @@ public static long writeNewKit(String kitRequestId, String addressIdTo, String e else { logger.info("Wrote new kit w/ dsm_kit_id " + kitRequestId, results.resultException); } - return (long) results.resultValue; + return Long.parseLong(String.valueOf(results.resultValue)); } // update kit with label trigger user and date diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java index df4c7c8bf..22d34202c 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java @@ -17,7 +17,6 @@ public static void setUp() { } @Test - @Ignore public void testExport() { final String index = "participants_structured.cmi.angio"; final String study = "angio"; @@ -33,7 +32,5 @@ public void testExport() { new TissueMigrator(index, study)); exportables.forEach(Exportable::export); -// Exportable exportable = new DynamicFieldsMappingMigrator("participants_structured.cmi.angio", "angio"); -// exportable.export(); } } \ No newline at end of file From 3a18d8925a46f5774abf199a257cefc6df8eff63 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 26 Jan 2022 11:52:17 +0400 Subject: [PATCH 345/405] DDP-7443 | change constructors of KitRequestShipping --- .../dsm/db/KitRequestShipping.java | 32 +++++++++++-------- .../broadinstitute/dsm/model/KitRequest.java | 6 ++-- .../dsm/model/KitUploadObject.java | 2 +- .../dsm/DDPRequestRouteTest.java | 13 ++++---- 4 files changed, 29 insertions(+), 24 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java index 7a17dcd1e..1368d4fe0 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java @@ -255,36 +255,39 @@ public KitRequestShipping() {} public KitRequestShipping(String collaboratorParticipantId, String kitTypeName, Long dsmKitRequestId, Long scanDate, Boolean error, Long receiveDate, Long deactivatedDate, String testResult, String upsTrackingStatus, String upsReturnStatus, String externalOrderStatus, String externalOrderNumber, Long externalOrderDate, Boolean careEvolve, String uploadReason) { - this(null, collaboratorParticipantId, null, null, null, kitTypeName, dsmKitRequestId, 0, null, null, + this(null, collaboratorParticipantId, null, null, null, kitTypeName, dsmKitRequestId, null, null, null, null, null, null, null, scanDate, error, null, receiveDate, - null, deactivatedDate, null, null, false, null, 0, null, externalOrderNumber, false, externalOrderStatus, null, testResult, + null, deactivatedDate, null, null, false, null, null, null, externalOrderNumber, false, externalOrderStatus, null, + testResult, upsTrackingStatus, upsReturnStatus, externalOrderDate, false, uploadReason, null, null, null); } public KitRequestShipping(String participantId, String collaboratorParticipantId, String dsmKitId, String realm, String trackingNumberTo, String receiveDateString, String hruid, String gender) { - this(participantId, collaboratorParticipantId, null, null, realm, null, 0, 0, null, null, - trackingNumberTo, null, null, null, 0, false, null, 0, - null, 0, null, dsmKitId, false, null, 0, null, null, false, null, null, null, null, null, 0, false, null, + this(participantId, collaboratorParticipantId, null, null, realm, null, null, null, null, null, + trackingNumberTo, null, null, null, null, false, null, null, + null, null, null, dsmKitId, false, null, null, null, null, false, null, null, null, null, null, null, false, null, receiveDateString, hruid, gender); } public KitRequestShipping(Long dsmKitRequestId, Long dsmKitId, String easypostToId, String easypostAddressId, Boolean error, String message) { this(null, null, null, null, null, null, dsmKitRequestId, dsmKitId, null, null, - null, null, null, null, 0, error, message, 0, - easypostAddressId, 0, null, null, false, easypostToId, 0, null, null, false, null, null, null, null, null, 0, false, null, + null, null, null, null, null, error, message, null, + easypostAddressId, null, null, null, false, easypostToId, null, null, null, false, null, null, null, null, null, null, + false, + null, null, null, null); } // shippingId = ddp_label !!! public KitRequestShipping(String participantId, String collaboratorParticipantId, String bspCollaboratorSampleId, String shippingId, String realm, - String kitTypeName, long dsmKitRequestId, long dsmKitId, String labelUrlTo, String labelUrlReturn, + String kitTypeName, Long dsmKitRequestId, Long dsmKitId, String labelUrlTo, String labelUrlReturn, String trackingNumberTo, String trackingReturnId, - String easypostTrackingToUrl, String trackingUrlReturn, long scanDate, boolean error, String message, - long receiveDate, String easypostAddressId, long deactivatedDate, String deactivationReason, - String kitLabel, boolean express, String easypostToId, long labelDate, String easypostShipmentStatus, - String externalOrderNumber, boolean noReturn, String externalOrderStatus, String createdBy, String testResult, - String upsTrackingStatus, String upsReturnStatus, long externalOrderDate, boolean careEvolve, String uploadReason, + String easypostTrackingToUrl, String trackingUrlReturn, Long scanDate, Boolean error, String message, + Long receiveDate, String easypostAddressId, Long deactivatedDate, String deactivationReason, + String kitLabel, Boolean express, String easypostToId, Long labelDate, String easypostShipmentStatus, + String externalOrderNumber, Boolean noReturn, String externalOrderStatus, String createdBy, String testResult, + String upsTrackingStatus, String upsReturnStatus, Long externalOrderDate, Boolean careEvolve, String uploadReason, String receiveDateString, String hruid, String gender) { super(dsmKitRequestId, participantId, null, shippingId, externalOrderNumber, null, externalOrderStatus, null, externalOrderDate); this.collaboratorParticipantId = collaboratorParticipantId; @@ -776,7 +779,8 @@ public static void deactivateKitRequest(long dsmKitRequestId, @NonNull String de } if (Objects.nonNull(ddpInstanceDto)) { - KitRequestShipping kitRequestShipping = new KitRequestShipping(dsmKitRequestId, null, null, null, null, null); + KitRequestShipping kitRequestShipping = new KitRequestShipping(); + kitRequestShipping.setDsmKitRequestId(dsmKitRequestId); kitRequestShipping.setDeactivationReason(deactivationReason); kitRequestShipping.setDeactivatedDate(deactivatedDate); diff --git a/src/main/java/org/broadinstitute/dsm/model/KitRequest.java b/src/main/java/org/broadinstitute/dsm/model/KitRequest.java index a2af4c89f..207e12392 100644 --- a/src/main/java/org/broadinstitute/dsm/model/KitRequest.java +++ b/src/main/java/org/broadinstitute/dsm/model/KitRequest.java @@ -17,7 +17,7 @@ public class KitRequest { @ColumnName(DBConstants.DSM_KIT_REQUEST_ID) - private long dsmKitRequestId; + private Long dsmKitRequestId; private String participantId; private String shortId; private String shippingId; @@ -37,10 +37,10 @@ public class KitRequest { public KitRequest() {} public KitRequest(String participantId, String shortId, DDPParticipant participant, String externalOrderNumber) { - this(0, participantId, shortId, null, externalOrderNumber, participant, null, null, null); + this(null, participantId, shortId, null, externalOrderNumber, participant, null, null, null); } - public KitRequest(long dsmKitRequestId, String participantId, String shortId, String shippingId, String externalOrderNumber, + public KitRequest(Long dsmKitRequestId, String participantId, String shortId, String shippingId, String externalOrderNumber, DDPParticipant participant, String externalOrderStatus, String externalKitName, Long externalOrderDate) { this.dsmKitRequestId = dsmKitRequestId; this.participantId = participantId; diff --git a/src/main/java/org/broadinstitute/dsm/model/KitUploadObject.java b/src/main/java/org/broadinstitute/dsm/model/KitUploadObject.java index 4f04b3c2f..bf1b7bae0 100644 --- a/src/main/java/org/broadinstitute/dsm/model/KitUploadObject.java +++ b/src/main/java/org/broadinstitute/dsm/model/KitUploadObject.java @@ -17,7 +17,7 @@ public class KitUploadObject extends KitRequest { public KitUploadObject(String externalOrderNumber, String ddpParticipantId, String shortId, String firstName, String lastName, String street1, String street2, String city, String state, String postalCode, String country, String phoneNumber) { - super(0, ddpParticipantId, shortId, null, externalOrderNumber, null, null, null, null); + super(null, ddpParticipantId, shortId, null, externalOrderNumber, null, null, null, null); this.firstName = firstName; this.lastName = lastName; this.street1 = street1; diff --git a/src/test/java/org/broadinstitute/dsm/DDPRequestRouteTest.java b/src/test/java/org/broadinstitute/dsm/DDPRequestRouteTest.java index 5b9718b60..b248ef1ff 100644 --- a/src/test/java/org/broadinstitute/dsm/DDPRequestRouteTest.java +++ b/src/test/java/org/broadinstitute/dsm/DDPRequestRouteTest.java @@ -105,10 +105,11 @@ public void checkForParticipant() throws Exception { response().withStatusCode(200).withBody(message.replaceAll("%1", test_participant_id).replaceAll("%2", "")) ); kitRequestTestList.add(new KitRequestShipping(test_participant_id, "TestProject_2", null, "FAKE_DSM_LABEL_UID", TEST_DDP, - "SALIVA", 1, 1, "https://easypost-files.s3-us-west-2.amazonaws.com/files/postage_label/20200214/8240f1b66535494a82b1ec0d566c3f0f.png", + "SALIVA", 1L, 1L, "https://easypost-files.s3-us-west-2.amazonaws" + + ".com/files/postage_label/20200214/8240f1b66535494a82b1ec0d566c3f0f.png", "", "794685038506", "9405536897846100551129", "https://track.easypost.com/djE6dHJrXzY4NGJmYzU3ZjM5OTQ1Zjg5MjEzOGRmMWVmMjI1NWZl", - null, 12, false, "", 12, null, 12, "so what", "mf_testLabel", false, "shp_f470591c3fb441a68dbb9b76ecf3bb3d", - 12, null, "44445", false, "NOT FOUND", null, null, null, null, 0, false, "STANDALONE", null, null, null)); + null, 12L, false, "", 12L, null, 12L, "so what", "mf_testLabel", false, "shp_f470591c3fb441a68dbb9b76ecf3bb3d", + 12L, null, "44445", false, "NOT FOUND", null, null, null, null, 0L, false, "STANDALONE", null, null, null)); addedDefaultKitRequest = true; counter = 1; } @@ -120,10 +121,10 @@ public void checkForParticipant() throws Exception { response().withStatusCode(200).withBody(message.replaceAll("%1", participant_id).replaceAll("%2", Integer.toString(counter))) ); kitRequestTestList.add(new KitRequestShipping(participant_id, "TestProject_2", null, "FAKE_DSM_LABEL_UID" + counter, TEST_DDP, - "SALIVA", 1, 1, "https://easypost-files.s3-us-west-2.amazonaws.com/files/postage_label/20200214/8240f1b66535494a82b1ec0d566c3f0f.png", + "SALIVA", 1L, 1L, "https://easypost-files.s3-us-west-2.amazonaws.com/files/postage_label/20200214/8240f1b66535494a82b1ec0d566c3f0f.png", "", "794685038506", "9405536897846100551129", "https://track.easypost.com/djE6dHJrXzY4NGJmYzU3ZjM5OTQ1Zjg5MjEzOGRmMWVmMjI1NWZl", - null, 12, false, "", 12, null, 12, "so what", "mf_testLabel", false, "shp_f470591c3fb441a68dbb9b76ecf3bb3d", - 12, null, "44445", false, "NOT FOUND", null, null, null, null, 0 ,false, null, null, null, null)); + null, 12L, false, "", 12L, null, 12L, "so what", "mf_testLabel", false, "shp_f470591c3fb441a68dbb9b76ecf3bb3d", + 12L, null, "44445", false, "NOT FOUND", null, null, null, null, 0L ,false, null, null, null, null)); counter++; } } From 8ad84f448008f7e0abf54a6a44116e66921217c1 Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 26 Jan 2022 12:23:23 +0400 Subject: [PATCH 346/405] DDP-7443 | rename ParticipantDataDto -> ParticipantData, refactor exportToES() method --- .../dsm/db/KitRequestShipping.java | 5 +- .../ddp/participant/ParticipantDataDao.java | 103 +++++++++--------- ...ipantDataDto.java => ParticipantData.java} | 12 +- .../export/WorkflowAndFamilyIdExporter.java | 18 +-- .../dsm/model/at/DefaultValues.java | 37 +++---- .../dsm/model/at/ReceiveKitRequest.java | 9 +- .../dsm/model/elastic/ESDsm.java | 6 +- .../migration/ParticipantDataMigrator.java | 4 +- .../model/familymember/AddFamilyMember.java | 15 ++- .../model/participant/ParticipantWrapper.java | 12 +- .../participant/ParticipantWrapperDto.java | 6 +- .../data/AddFamilyMemberPayload.java | 11 +- .../participant/data/FamilyMemberDetails.java | 4 - .../participant/data/ParticipantData.java | 33 +++--- .../dsm/pubsub/WorkflowStatusUpdate.java | 28 ++--- .../dsm/util/ParticipantUtil.java | 16 +-- .../broadinstitute/dsm/ParticipantTest.java | 18 +-- .../dsm/model/elastic/UtilTest.java | 8 +- .../participant/ParticipantWrapperTest.java | 4 +- .../participant/data/ParticipantDataTest.java | 32 +++--- .../dsm/pubsub/UpdateWorkflowStatusTest.java | 16 +-- .../AddFamilyMemberRouteTest.java | 23 ++-- 22 files changed, 203 insertions(+), 217 deletions(-) rename src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/{ParticipantDataDto.java => ParticipantData.java} (94%) diff --git a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java index 1368d4fe0..eec487add 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java @@ -29,6 +29,7 @@ import org.broadinstitute.dsm.model.elastic.export.painless.UpsertPainless; import org.broadinstitute.dsm.statics.ApplicationConfigConstants; import org.broadinstitute.dsm.statics.DBConstants; +import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.statics.QueryExtension; import org.broadinstitute.dsm.util.DBUtil; import org.broadinstitute.dsm.util.EasyPostUtil; @@ -1444,8 +1445,8 @@ public static void exportToES(Object source, DDPInstanceDto ddpInstanceDto, Stri String fieldName, Object fieldValue) { Generator paramsGenerator = new ParamsGenerator(source, ddpInstanceDto.getInstanceName()); ScriptBuilder scriptBuilder = new NestedScriptBuilder(paramsGenerator.getPropertyName(), uniqueIdentifier); - MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder(fieldName, fieldValue); - NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder("dsm.kitRequestShipping", matchQueryBuilder, ScoreMode.Avg); + MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder(String.join(".",ESObjectConstants.DSM, paramsGenerator.getPropertyName(), fieldName), fieldValue); + NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder(String.join(".",ESObjectConstants.DSM, paramsGenerator.getPropertyName()), matchQueryBuilder, ScoreMode.Avg); UpsertPainless upsertPainless = new UpsertPainless(paramsGenerator, ddpInstanceDto.getEsParticipantIndex(), scriptBuilder, nestedQueryBuilder); upsertPainless.export(); } diff --git a/src/main/java/org/broadinstitute/dsm/db/dao/ddp/participant/ParticipantDataDao.java b/src/main/java/org/broadinstitute/dsm/db/dao/ddp/participant/ParticipantDataDao.java index f268d4052..baf3c1033 100644 --- a/src/main/java/org/broadinstitute/dsm/db/dao/ddp/participant/ParticipantDataDao.java +++ b/src/main/java/org/broadinstitute/dsm/db/dao/ddp/participant/ParticipantDataDao.java @@ -7,7 +7,6 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -16,11 +15,11 @@ import org.broadinstitute.ddp.db.SimpleResult; import org.broadinstitute.dsm.db.dao.Dao; -import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; +import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class ParticipantDataDao implements Dao { +public class ParticipantDataDao implements Dao { private static final Logger logger = LoggerFactory.getLogger(ParticipantDataDao.class); @@ -81,16 +80,16 @@ public class ParticipantDataDao implements Dao { @Override - public int create(ParticipantDataDto participantDataDto) { + public int create(ParticipantData participantData) { SimpleResult simpleResult = inTransaction(conn -> { SimpleResult dbVals = new SimpleResult(-1); try (PreparedStatement stmt = conn.prepareStatement(SQL_INSERT_DATA_TO_PARTICIPANT_DATA, Statement.RETURN_GENERATED_KEYS)) { - stmt.setString(1, participantDataDto.getDdpParticipantId().orElse("")); - stmt.setInt(2, participantDataDto.getDdpInstanceId()); - stmt.setString(3, participantDataDto.getFieldTypeId().orElse("")); - stmt.setString(4, participantDataDto.getData().orElse("")); - stmt.setLong(5, participantDataDto.getLastChanged()); - stmt.setString(6, participantDataDto.getChangedBy().orElse("SYSTEM")); + stmt.setString(1, participantData.getDdpParticipantId().orElse("")); + stmt.setInt(2, participantData.getDdpInstanceId()); + stmt.setString(3, participantData.getFieldTypeId().orElse("")); + stmt.setString(4, participantData.getData().orElse("")); + stmt.setLong(5, participantData.getLastChanged()); + stmt.setString(6, participantData.getChangedBy().orElse("SYSTEM")); stmt.executeUpdate(); try (ResultSet rs = stmt.getGeneratedKeys()) { if (rs.next()) { @@ -130,14 +129,14 @@ public int delete(int id) { } @Override - public Optional get(long id) { + public Optional get(long id) { SimpleResult results = inTransaction((conn) -> { SimpleResult execResult = new SimpleResult(); try (PreparedStatement stmt = conn.prepareStatement(SQL_PARTICIPANT_DATA_BY_ID)) { stmt.setLong(1, id); try(ResultSet rs = stmt.executeQuery()) { if (rs.next()) { - execResult.resultValue = new ParticipantDataDto.Builder() + execResult.resultValue = new ParticipantData.Builder() .withParticipantDataId(rs.getInt(PARTICIPANT_DATA_ID)) .withDdpParticipantId(rs.getString(DDP_PARTICIPANT_ID)) .withDdpInstanceId(rs.getInt(DDP_INSTANCE_ID)) @@ -158,17 +157,17 @@ public Optional get(long id) { throw new RuntimeException("Error getting participant data with " + id, results.resultException); } - return Optional.ofNullable((ParticipantDataDto) results.resultValue); + return Optional.ofNullable((ParticipantData) results.resultValue); } - public int updateParticipantDataColumn(ParticipantDataDto participantDataDto) { + public int updateParticipantDataColumn(ParticipantData participantData) { SimpleResult result = inTransaction((conn) -> { SimpleResult execResult = new SimpleResult(); try (PreparedStatement stmt = conn.prepareStatement(SQL_UPDATE_DATA_TO_PARTICIPANT_DATA)) { - stmt.setString(1, participantDataDto.getData().orElse("")); - stmt.setLong(2, participantDataDto.getLastChanged()); - stmt.setString(3, participantDataDto.getChangedBy().orElse("SYSTEM")); - stmt.setInt(4, participantDataDto.getParticipantDataId()); + stmt.setString(1, participantData.getData().orElse("")); + stmt.setLong(2, participantData.getLastChanged()); + stmt.setString(3, participantData.getChangedBy().orElse("SYSTEM")); + stmt.setInt(4, participantData.getParticipantDataId()); execResult.resultValue = stmt.executeUpdate(); } catch (SQLException sqle) { execResult.resultException = sqle; @@ -177,24 +176,24 @@ public int updateParticipantDataColumn(ParticipantDataDto participantDataDto) { }); if (result.resultException != null) { throw new RuntimeException(String.format("Could not update data for participant data with id: %s for participant with guid: %s", - participantDataDto.getParticipantDataId(), participantDataDto.getDdpParticipantId())); + participantData.getParticipantDataId(), participantData.getDdpParticipantId())); } logger.info(String.format("Updated data for participant data with id: %s for participant with guid: %s", - participantDataDto.getParticipantDataId(), participantDataDto.getDdpParticipantId())); + participantData.getParticipantDataId(), participantData.getDdpParticipantId())); return (int) result.resultValue; } - public List getParticipantDataByParticipantId(String participantId) { - List participantDataDtoList = new ArrayList<>(); + public List getParticipantDataByParticipantId(String participantId) { + List participantDataList = new ArrayList<>(); SimpleResult results = inTransaction((conn) -> { SimpleResult execResult = new SimpleResult(); try (PreparedStatement stmt = conn.prepareStatement(SQL_PARTICIPANT_DATA_BY_PARTICIPANT_ID)) { stmt.setString(1, participantId); try(ResultSet rs = stmt.executeQuery()) { while (rs.next()) { - participantDataDtoList.add( - new ParticipantDataDto.Builder() + participantDataList.add( + new ParticipantData.Builder() .withParticipantDataId(rs.getInt(PARTICIPANT_DATA_ID)) .withDdpParticipantId(rs.getString(DDP_PARTICIPANT_ID)) .withDdpInstanceId(rs.getInt(DDP_INSTANCE_ID)) @@ -216,20 +215,20 @@ public List getParticipantDataByParticipantId(String partici throw new RuntimeException("Error getting participant data with " + participantId, results.resultException); } - return participantDataDtoList; + return participantDataList; } - public Map> getParticipantDataByParticipantIds(List participantIds) { + public Map> getParticipantDataByParticipantIds(List participantIds) { String sqlWithInClause = SQL_GET_PARTICIPANT_DATA_BY_PARTICIPANT_IDS.replace("?", participantIds.stream().collect(Collectors.joining("','", "'", "'"))); - Map> participantDatasByParticipantIds = new HashMap<>(); + Map> participantDatasByParticipantIds = new HashMap<>(); SimpleResult results = inTransaction((conn) -> { SimpleResult execResult = new SimpleResult(); try (PreparedStatement stmt = conn.prepareStatement(sqlWithInClause)) { try(ResultSet rs = stmt.executeQuery()) { while (rs.next()) { - ArrayList value = new ArrayList<>( - List.of(new ParticipantDataDto.Builder() + ArrayList value = new ArrayList<>( + List.of(new ParticipantData.Builder() .withParticipantDataId(rs.getInt(PARTICIPANT_DATA_ID)) .withDdpParticipantId(rs.getString(DDP_PARTICIPANT_ID)) .withDdpInstanceId(rs.getInt(DDP_INSTANCE_ID)) @@ -258,16 +257,16 @@ public Map> getParticipantDataByParticipantIds( return participantDatasByParticipantIds; } - public Map> getParticipantDataByRealm(String realm) { - Map> participantDataByRealm = new HashMap<>(); + public Map> getParticipantDataByRealm(String realm) { + Map> participantDataByRealm = new HashMap<>(); SimpleResult results = inTransaction((conn) -> { SimpleResult execResult = new SimpleResult(); try (PreparedStatement stmt = conn.prepareStatement(SQL_SELECT_PARTICIPANTS_BY_REALM)) { stmt.setString(1, realm); try(ResultSet rs = stmt.executeQuery()) { while (rs.next()) { - ArrayList value = new ArrayList<>( - List.of(new ParticipantDataDto.Builder() + ArrayList value = new ArrayList<>( + List.of(new ParticipantData.Builder() .withParticipantDataId(rs.getInt(PARTICIPANT_DATA_ID)) .withDdpParticipantId(rs.getString(DDP_PARTICIPANT_ID)) .withDdpInstanceId(rs.getInt(DDP_INSTANCE_ID)) @@ -295,16 +294,16 @@ public Map> getParticipantDataByRealm(String re } - public Map> getParticipantDataByInstanceIdAndFilterQuery(int ddpInstanceId, String filterQuery) { - Map> participantDatasByParticipantIds = new HashMap<>(); + public Map> getParticipantDataByInstanceIdAndFilterQuery(int ddpInstanceId, String filterQuery) { + Map> participantDatasByParticipantIds = new HashMap<>(); SimpleResult results = inTransaction((conn) -> { SimpleResult execResult = new SimpleResult(); try (PreparedStatement stmt = conn.prepareStatement(SQL_ALL_PARTICIPANT_DATA + BY_INSTANCE_ID + filterQuery)) { stmt.setInt(1, ddpInstanceId); try(ResultSet rs = stmt.executeQuery()) { while (rs.next()) { - ArrayList value = new ArrayList<>( - List.of(new ParticipantDataDto.Builder() + ArrayList value = new ArrayList<>( + List.of(new ParticipantData.Builder() .withParticipantDataId(rs.getInt(PARTICIPANT_DATA_ID)) .withDdpParticipantId(rs.getString(DDP_PARTICIPANT_ID)) .withDdpInstanceId(rs.getInt(DDP_INSTANCE_ID)) @@ -333,15 +332,15 @@ public Map> getParticipantDataByInstanceIdAndFi return participantDatasByParticipantIds; } - public List getAllParticipantData() { - List participantDataDtoList = new ArrayList<>(); + public List getAllParticipantData() { + List participantDataList = new ArrayList<>(); SimpleResult results = inTransaction((conn) -> { SimpleResult execResult = new SimpleResult(); try (PreparedStatement stmt = conn.prepareStatement(SQL_ALL_PARTICIPANT_DATA)) { try(ResultSet rs = stmt.executeQuery()) { while (rs.next()) { - participantDataDtoList.add( - new ParticipantDataDto.Builder() + participantDataList.add( + new ParticipantData.Builder() .withParticipantDataId(rs.getInt(PARTICIPANT_DATA_ID)) .withDdpParticipantId(rs.getString(DDP_PARTICIPANT_ID)) .withDdpInstanceId(rs.getInt(DDP_INSTANCE_ID)) @@ -362,19 +361,19 @@ public List getAllParticipantData() { if (results.resultException != null) { throw new RuntimeException("Error getting participant data ", results.resultException); } - return participantDataDtoList; + return participantDataList; } - public List getParticipantDataByInstanceId(int instanceId) { - List participantDataDtoList = new ArrayList<>(); + public List getParticipantDataByInstanceId(int instanceId) { + List participantDataList = new ArrayList<>(); SimpleResult results = inTransaction((conn) -> { SimpleResult execResult = new SimpleResult(); try (PreparedStatement stmt = conn.prepareStatement(SQL_ALL_PARTICIPANT_DATA + BY_INSTANCE_ID)) { stmt.setInt(1, instanceId); try(ResultSet rs = stmt.executeQuery()) { while (rs.next()) { - participantDataDtoList.add( - new ParticipantDataDto.Builder() + participantDataList.add( + new ParticipantData.Builder() .withParticipantDataId(rs.getInt(PARTICIPANT_DATA_ID)) .withDdpParticipantId(rs.getString(DDP_PARTICIPANT_ID)) .withDdpInstanceId(rs.getInt(DDP_INSTANCE_ID)) @@ -395,20 +394,20 @@ public List getParticipantDataByInstanceId(int instanceId) { if (results.resultException != null) { throw new RuntimeException("Error getting participant data ", results.resultException); } - return participantDataDtoList; + return participantDataList; } - public List getParticipantDataByInstanceIdAndQueryAddition(int instanceId, String queryAddition) { - List participantDataDtoList = new ArrayList<>(); + public List getParticipantDataByInstanceIdAndQueryAddition(int instanceId, String queryAddition) { + List participantDataList = new ArrayList<>(); SimpleResult results = inTransaction((conn) -> { SimpleResult execResult = new SimpleResult(); try (PreparedStatement stmt = conn.prepareStatement(SQL_ALL_PARTICIPANT_DATA + BY_INSTANCE_ID + queryAddition)) { stmt.setInt(1, instanceId); try(ResultSet rs = stmt.executeQuery()) { while (rs.next()) { - participantDataDtoList.add( - new ParticipantDataDto.Builder() + participantDataList.add( + new ParticipantData.Builder() .withParticipantDataId(rs.getInt(PARTICIPANT_DATA_ID)) .withDdpParticipantId(rs.getString(DDP_PARTICIPANT_ID)) .withDdpInstanceId(rs.getInt(DDP_INSTANCE_ID)) @@ -429,7 +428,7 @@ public List getParticipantDataByInstanceIdAndQueryAddition(i if (results.resultException != null) { throw new RuntimeException("Error getting participant data ", results.resultException); } - return participantDataDtoList; + return participantDataList; } } diff --git a/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java b/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantData.java similarity index 94% rename from src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java rename to src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantData.java index 8ffd1d72a..00847a39a 100644 --- a/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantDataDto.java +++ b/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantData.java @@ -8,9 +8,7 @@ import com.fasterxml.jackson.annotation.*; import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; -import com.google.gson.annotations.SerializedName; import com.google.gson.reflect.TypeToken; import lombok.Setter; import org.apache.commons.lang3.StringUtils; @@ -27,7 +25,7 @@ columnPrefix = "") @JsonIgnoreProperties(ignoreUnknown = true) @Setter -public class ParticipantDataDto { +public class ParticipantData { private static final Gson gson = new Gson(); @@ -76,7 +74,7 @@ public Map getDynamicFields() { } } - public ParticipantDataDto() {} + public ParticipantData() {} @JsonIgnore private long lastChanged; @@ -134,7 +132,7 @@ public Optional getChangedBy() { return Optional.ofNullable(changedBy); } - private ParticipantDataDto(Builder builder) { + private ParticipantData(Builder builder) { this.participantDataId = builder.participantDataId; this.ddpParticipantId = builder.ddpParticipantId; this.ddpInstanceId = builder.ddpInstanceId; @@ -188,8 +186,8 @@ public Builder withChangedBy(String changedBy) { return this; } - public ParticipantDataDto build() { - return new ParticipantDataDto(this); + public ParticipantData build() { + return new ParticipantData(this); } diff --git a/src/main/java/org/broadinstitute/dsm/export/WorkflowAndFamilyIdExporter.java b/src/main/java/org/broadinstitute/dsm/export/WorkflowAndFamilyIdExporter.java index cba6dc237..70118c02d 100644 --- a/src/main/java/org/broadinstitute/dsm/export/WorkflowAndFamilyIdExporter.java +++ b/src/main/java/org/broadinstitute/dsm/export/WorkflowAndFamilyIdExporter.java @@ -5,7 +5,7 @@ import org.broadinstitute.dsm.db.DDPInstance; import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; import org.broadinstitute.dsm.db.dao.settings.FieldSettingsDao; -import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; +import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData; import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; import org.broadinstitute.dsm.model.Value; import org.broadinstitute.dsm.model.elastic.ESProfile; @@ -43,7 +43,7 @@ public void export(DDPInstance instance, boolean clearBeforeUpdate) { logger.info("Found {} columns for elastic workflow export for instanceId {}", workflowColumnNames.size(), instanceId); // We use a queue here so we can pop things off as we go to save on memory. - ArrayDeque> queue = new ArrayDeque<>(participantDataDao + ArrayDeque> queue = new ArrayDeque<>(participantDataDao .getParticipantDataByInstanceId(instanceId).stream() .collect(Collectors.groupingBy(dto -> dto.getDdpParticipantId().orElse(""))) .values()); @@ -76,10 +76,10 @@ private List findWorkFlowColumnNames(int instanceId) { } private void checkWorkflowNamesAndExport(DDPInstance instance, List workflowColumnNames, - Deque> queue, boolean clearBeforeUpdate) { + Deque> queue, boolean clearBeforeUpdate) { String index = instance.getParticipantIndexES(); while (!queue.isEmpty()) { - List familyGroup = queue.pop(); + List familyGroup = queue.pop(); if (familyGroup == null || familyGroup.isEmpty()) { continue; } @@ -112,19 +112,19 @@ private void checkWorkflowNamesAndExport(DDPInstance instance, List work editor.clear(); } - ParticipantDataDto applicantDataDto = ParticipantUtil.findApplicantData(guidOrAltPid, familyGroup); + ParticipantData applicantDataDto = ParticipantUtil.findApplicantData(guidOrAltPid, familyGroup); if (applicantDataDto == null) { logger.error("Somehow there's no applicant for guid/altpid: {}, continuing with export", guidOrAltPid); continue; } String familyId = null; - for (ParticipantDataDto participantDataDto : familyGroup) { - Map dataMap = participantDataDto.getDataMap(); + for (ParticipantData participantData : familyGroup) { + Map dataMap = participantData.getDataMap(); if (dataMap == null) { continue; } - processWorkflows(instance, workflowColumnNames, guidOrAltPid, participantDataDto, applicantDataDto, profile, editor); + processWorkflows(instance, workflowColumnNames, guidOrAltPid, participantData, applicantDataDto, profile, editor); if (dataMap.containsKey(FamilyMemberConstants.FAMILY_ID)) { familyId = dataMap.get(FamilyMemberConstants.FAMILY_ID); } @@ -143,7 +143,7 @@ private void checkWorkflowNamesAndExport(DDPInstance instance, List work } private void processWorkflows(DDPInstance instance, List workflowColumnNames, String guidOrAltPid, - ParticipantDataDto participantData, ParticipantDataDto applicantData, + ParticipantData participantData, ParticipantData applicantData, ESProfile applicantProfile, WorkflowsEditor editor) { Map dataMap = participantData.getDataMap(); if (participantData.getFieldTypeId().orElse("").equals(RGP_PARTICIPANTS)) { diff --git a/src/main/java/org/broadinstitute/dsm/model/at/DefaultValues.java b/src/main/java/org/broadinstitute/dsm/model/at/DefaultValues.java index 2719250e6..77a49afcf 100644 --- a/src/main/java/org/broadinstitute/dsm/model/at/DefaultValues.java +++ b/src/main/java/org/broadinstitute/dsm/model/at/DefaultValues.java @@ -10,12 +10,11 @@ import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; import org.broadinstitute.dsm.db.dao.settings.FieldSettingsDao; import org.broadinstitute.dsm.db.dto.bookmark.BookmarkDto; -import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; +import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData; import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; import org.broadinstitute.dsm.model.elastic.ESProfile; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.settings.field.FieldSettings; -import org.broadinstitute.dsm.model.participant.data.ParticipantData; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,13 +42,13 @@ public class DefaultValues { public static final String AT_GENOMIC_ID = "at_genomic_id"; private Dao dataAccess; - private Map> participantData; + private Map> participantData; private List participantESData; private DDPInstance instance; private String queryAddition; private ParticipantDataDao participantDataDao; - public DefaultValues(Map> participantData, + public DefaultValues(Map> participantData, List participantESData, @NonNull DDPInstance instance, String queryAddition) { this.participantData = participantData; this.participantESData = participantESData; @@ -58,7 +57,7 @@ public DefaultValues(Map> participantData, this.participantDataDao = new ParticipantDataDao(); } - public Map> addDefaultValues() { + public Map> addDefaultValues() { if (participantESData == null) { logger.warn("Could not update default values, participant ES data is null"); return participantData; @@ -68,7 +67,7 @@ public Map> addDefaultValues() { participantESData.stream().collect(Collectors.toMap(ElasticSearchParticipantDto::getParticipantId, Function.identity())); for (Map.Entry entry: participantEsDataByParticipantId.entrySet()) { String ddpParticipantId = entry.getKey(); - List participantDataList = participantData.get(ddpParticipantId); + List participantDataList = participantData.get(ddpParticipantId); if (participantDataList == null) continue; if (!hasParticipantDataGenomicId(participantDataList) && isSelfOrDependentParticipant(participantDataList)) { @@ -89,13 +88,13 @@ public Map> addDefaultValues() { if (addedNewParticipantData) { //participant data was added, getting new list of data if (StringUtils.isNotBlank(queryAddition)) { - List participantDataByInstanceIdAndQueryAddition = + List participantDataByInstanceIdAndQueryAddition = participantDataDao.getParticipantDataByInstanceIdAndQueryAddition(Integer.parseInt(instance.getDdpInstanceId()), queryAddition); participantData = getParticipantDataWithParticipantId(participantDataByInstanceIdAndQueryAddition); } else { - List participantDataByInstanceId = + List participantDataByInstanceId = participantDataDao.getParticipantDataByInstanceId(Integer.parseInt(instance.getDdpInstanceId())); participantData = getParticipantDataWithParticipantId(participantDataByInstanceId); } @@ -103,7 +102,7 @@ public Map> addDefaultValues() { return participantData; } - private Map> getParticipantDataWithParticipantId(List participantDataByInstanceId) { + private Map> getParticipantDataWithParticipantId(List participantDataByInstanceId) { return participantDataByInstanceId.stream() .collect(Collectors.toMap( pDataDto -> pDataDto.getDdpParticipantId().orElse(""), @@ -115,7 +114,7 @@ private Map> getParticipantDataWithParticipantI )); } - private boolean hasParticipantDataGenomicId(List participantDataList) { + private boolean hasParticipantDataGenomicId(List participantDataList) { if (participantDataList.isEmpty()) return false; return participantDataList.stream() .anyMatch(participantData -> { @@ -124,7 +123,7 @@ private boolean hasParticipantDataGenomicId(List participant }); } - private boolean isSelfOrDependentParticipant(List participantDataList) { + private boolean isSelfOrDependentParticipant(List participantDataList) { if (participantDataList.isEmpty()) return false; return participantDataList.stream() .anyMatch(pData -> { @@ -135,7 +134,7 @@ private boolean isSelfOrDependentParticipant(List participan }); } - private boolean hasExitedStatusDefaultValue(List participantDataList) { + private boolean hasExitedStatusDefaultValue(List participantDataList) { if (participantDataList.isEmpty()) return false; return participantDataList.stream() .anyMatch(participantData -> { @@ -144,7 +143,7 @@ private boolean hasExitedStatusDefaultValue(List participant }); } - private Optional getParticipantGenomicFieldData(List participantDataList) { + private Optional getParticipantGenomicFieldData(List participantDataList) { if (participantDataList.isEmpty()) return Optional.empty(); return participantDataList.stream() .filter(participantData -> FIELD_TYPE_ID.equals(participantData.getFieldTypeId().orElse(""))) @@ -152,7 +151,7 @@ private Optional getParticipantGenomicFieldData(List dataMap = GSON.fromJson(pData.getData().orElse(""), new TypeToken>() {}.getType()); if (dataMap.containsKey(GENOME_STUDY_CPT_ID)) return false; dataMap.put(GENOME_STUDY_CPT_ID, PREFIX.concat(getGenomicIdValue(hruid))); @@ -176,14 +175,14 @@ private boolean insertGenomicIdForParticipant(String ddpParticipantId, String hr return insertParticipantData(Map.of(GENOME_STUDY_CPT_ID, PREFIX.concat(getGenomicIdValue(hruid))), ddpParticipantId, FIELD_TYPE_ID); } - private Optional getParticipantExitFieldData(List participantDataList) { + private Optional getParticipantExitFieldData(List participantDataList) { if (participantDataList.isEmpty()) return Optional.empty(); return participantDataList.stream() .filter(participantData -> AT_PARTICIPANT_EXIT.equals(participantData.getFieldTypeId().orElse(""))) .findFirst(); } - private boolean insertExitStatusIfNotExistsInData(String ddpParticipantId, ParticipantDataDto pData) { + private boolean insertExitStatusIfNotExistsInData(String ddpParticipantId, ParticipantData pData) { Map dataMap = GSON.fromJson(pData.getData().orElse(""), new TypeToken>() {}.getType()); if (dataMap.containsKey(EXITSTATUS)) return false; dataMap.put(EXITSTATUS, getDefaultExitStatus()); @@ -195,16 +194,16 @@ private boolean insertExitStatusForParticipant(String ddpParticipantId) { return insertParticipantData(Map.of(EXITSTATUS, datstatExitReasonDefaultOption), ddpParticipantId, AT_PARTICIPANT_EXIT); } - private boolean updateParticipantData(String ddpParticipantId, ParticipantDataDto pData, Map dataMap) { + private boolean updateParticipantData(String ddpParticipantId, ParticipantData pData, Map dataMap) { this.setDataAccess(new ParticipantDataDao()); - ParticipantData participantData = new ParticipantData(dataAccess); + org.broadinstitute.dsm.model.participant.data.ParticipantData participantData = new org.broadinstitute.dsm.model.participant.data.ParticipantData(dataAccess); participantData.setData(ddpParticipantId, Integer.parseInt(instance.getDdpInstanceId()), pData.getFieldTypeId().orElse(""), dataMap); return participantData.updateParticipantData(pData.getParticipantDataId(), "SYSTEM"); } private boolean insertParticipantData(Map data, String ddpParticipantId, String fieldTypeId) { this.setDataAccess(new ParticipantDataDao()); - ParticipantData participantData = new ParticipantData(dataAccess); + org.broadinstitute.dsm.model.participant.data.ParticipantData participantData = new org.broadinstitute.dsm.model.participant.data.ParticipantData(dataAccess); participantData.setData(ddpParticipantId, Integer.parseInt(instance.getDdpInstanceId()), fieldTypeId, data); try { diff --git a/src/main/java/org/broadinstitute/dsm/model/at/ReceiveKitRequest.java b/src/main/java/org/broadinstitute/dsm/model/at/ReceiveKitRequest.java index ad16ad8c4..c24cb4afe 100644 --- a/src/main/java/org/broadinstitute/dsm/model/at/ReceiveKitRequest.java +++ b/src/main/java/org/broadinstitute/dsm/model/at/ReceiveKitRequest.java @@ -6,10 +6,9 @@ import org.apache.commons.lang3.StringUtils; import org.broadinstitute.ddp.db.SimpleResult; import org.broadinstitute.dsm.db.KitRequestShipping; -import org.broadinstitute.dsm.db.ParticipantData; import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; -import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; +import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData; import org.broadinstitute.dsm.util.NotificationUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,7 +34,7 @@ public class ReceiveKitRequest { private static final String NOTIFICATION_MESSAGE = "Sample GENOME_STUDY_SPIT_KIT_BARCODE has been received at the Broad Institute as of GENOME_STUDY_DATE_RECEIVED."; public static boolean receiveATKitRequest(@NonNull NotificationUtil notificationUtil, @NonNull String mfBarcode) { - ParticipantData participantData = SearchKitRequest.findATKitRequest(mfBarcode); + org.broadinstitute.dsm.db.ParticipantData participantData = SearchKitRequest.findATKitRequest(mfBarcode); if (participantData != null && StringUtils.isNotBlank(participantData.getData())) { Map data = new Gson().fromJson(participantData.getData(), new TypeToken>(){}.getType()); long now = Instant.now().toEpochMilli(); @@ -84,12 +83,12 @@ private static boolean updateData(@NonNull String data, @NonNull long participan throw new RuntimeException("Error setting AT kit to received with id " + participantDataId, results.resultException); } - ParticipantDataDto participantDataDto = new ParticipantDataDto.Builder() + ParticipantData participantData = new ParticipantData.Builder() .withData(data) .withParticipantDataId((int) participantDataId) .build(); - KitRequestShipping.exportToES(participantDataDto, ddpInstanceDto, "participantDataId", "participantDataId", participantDataId); + KitRequestShipping.exportToES(participantData, ddpInstanceDto, "participantDataId", "participantDataId", participantDataId); return true; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java index 0ad4d8169..db5cc63f1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/ESDsm.java @@ -9,7 +9,7 @@ import lombok.Getter; import lombok.Setter; import org.broadinstitute.dsm.db.*; -import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; +import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData; import org.broadinstitute.dsm.statics.ESObjectConstants; @Getter @@ -50,7 +50,7 @@ public class ESDsm { List oncHistoryDetail; @SerializedName(ESObjectConstants.PARTICIPANT_DATA) - List participantData; + List participantData; @SerializedName(ESObjectConstants.PARTICIPANT) Participant participant; @@ -76,7 +76,7 @@ public List getOncHistoryDetail() { return oncHistoryDetail; } - public List getParticipantData() { + public List getParticipantData() { if (participantData == null) participantData = Collections.emptyList(); return participantData; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java index 660163231..a51d1b996 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/migration/ParticipantDataMigrator.java @@ -5,7 +5,7 @@ import java.util.Map; import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; -import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; +import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData; import org.broadinstitute.dsm.statics.ESObjectConstants; public class ParticipantDataMigrator extends BaseCollectionMigrator { @@ -19,7 +19,7 @@ public ParticipantDataMigrator(String index, String realm) { @Override protected Map getDataByRealm() { - Map> participantDataByRealm = participantDataDao.getParticipantDataByRealm(realm); + Map> participantDataByRealm = participantDataDao.getParticipantDataByRealm(realm); return (Map) participantDataByRealm; } } diff --git a/src/main/java/org/broadinstitute/dsm/model/familymember/AddFamilyMember.java b/src/main/java/org/broadinstitute/dsm/model/familymember/AddFamilyMember.java index a8164110e..d6a9c40ab 100644 --- a/src/main/java/org/broadinstitute/dsm/model/familymember/AddFamilyMember.java +++ b/src/main/java/org/broadinstitute/dsm/model/familymember/AddFamilyMember.java @@ -13,14 +13,13 @@ import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; import org.broadinstitute.dsm.db.dao.settings.FieldSettingsDao; import org.broadinstitute.dsm.db.dao.user.UserDao; -import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; +import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData; import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; import org.broadinstitute.dsm.export.WorkflowForES; import org.broadinstitute.dsm.model.Study; import org.broadinstitute.dsm.model.participant.data.AddFamilyMemberPayload; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import org.broadinstitute.dsm.model.participant.data.FamilyMemberDetails; -import org.broadinstitute.dsm.model.participant.data.ParticipantData; import org.broadinstitute.dsm.model.rgp.RgpAddFamilyMember; import org.broadinstitute.dsm.model.settings.field.FieldSettings; import org.broadinstitute.dsm.util.ElasticSearchUtil; @@ -33,7 +32,7 @@ public class AddFamilyMember { private static final Logger logger = LoggerFactory.getLogger(AddFamilyMember.class); protected AddFamilyMemberPayload addFamilyMemberPayload; - protected ParticipantData participantData; + protected org.broadinstitute.dsm.model.participant.data.ParticipantData participantData; protected int ddpInstanceId; protected DDPInstanceDao ddpInstanceDao; protected String studyGuid; @@ -41,7 +40,7 @@ public class AddFamilyMember { protected AddFamilyMember(AddFamilyMemberPayload addFamilyMemberPayload) { this.addFamilyMemberPayload = Objects.requireNonNull(addFamilyMemberPayload); - this.participantData = new ParticipantData(); + this.participantData = new org.broadinstitute.dsm.model.participant.data.ParticipantData(); ddpInstanceDao = new DDPInstanceDao(); studyGuid = addFamilyMemberPayload.getRealm().orElseThrow(); ddpParticipantId = addFamilyMemberPayload.getParticipantId().orElseThrow(); @@ -60,7 +59,7 @@ public long add() { protected void prepareFamilyMemberData() { FamilyMemberDetails familyMemberDetails = addFamilyMemberPayload.getData().orElseThrow(); - String fieldTypeId = studyGuid + ParticipantData.FIELD_TYPE_PARTICIPANTS; + String fieldTypeId = studyGuid + org.broadinstitute.dsm.model.participant.data.ParticipantData.FIELD_TYPE_PARTICIPANTS; participantData.setDdpParticipantId(ddpParticipantId); participantData.setDdpInstanceId(ddpInstanceId); participantData.setFieldTypeId(fieldTypeId); @@ -78,10 +77,10 @@ protected void copyProbandData() { if (!isCopyProband || StringUtils.isBlank(addFamilyMemberPayload.getParticipantId().orElse(""))) return; Map participantDataData = participantData.getData(); if (Objects.isNull(participantDataData)) throw new NoSuchElementException(); - List participantDataByParticipantId = + List participantDataByParticipantId = participantData.getParticipantDataByParticipantId(addFamilyMemberPayload.getParticipantId().orElse("")); - Optional maybeProbandData = participantData.findProband(participantDataByParticipantId); - Optional maybeParticipantData = maybeProbandData.map(ParticipantData::parseDto); + Optional maybeProbandData = participantData.findProband(participantDataByParticipantId); + Optional maybeParticipantData = maybeProbandData.map(org.broadinstitute.dsm.model.participant.data.ParticipantData::parseDto); maybeParticipantData.ifPresent(participantData -> participantData.getData().forEach(participantDataData::putIfAbsent)); } diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java index 46466c73f..72375f1f8 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java @@ -8,7 +8,7 @@ import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; -import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; +import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData; import org.broadinstitute.dsm.model.elastic.ESProfile; import org.broadinstitute.dsm.model.elastic.filter.query.DsmAbstractQueryBuilder; import org.broadinstitute.dsm.model.elastic.filter.FilterParser; @@ -46,7 +46,7 @@ public class ParticipantWrapper { private Map> abstractionActivities = new HashMap<>(); private Map> abstractionSummary = new HashMap<>(); private Map> proxiesByParticipantIds = new HashMap<>(); - private Map> participantData = new HashMap<>(); + private Map> participantData = new HashMap<>(); public ParticipantWrapper(ParticipantWrapperPayload participantWrapperPayload, ElasticSearchable elasticSearchable) { this.participantWrapperPayload = Objects.requireNonNull(participantWrapperPayload); @@ -142,8 +142,8 @@ private void fetchAndPrepareData(DDPInstance ddpInstance) { participantData = getParticipantDataFromEsData(); } - private Map> getParticipantDataFromEsData() { - Map> participantDataByParticipantId = new HashMap<>(); + private Map> getParticipantDataFromEsData() { + Map> participantDataByParticipantId = new HashMap<>(); for (ElasticSearchParticipantDto elasticSearchParticipantDto: esData.getEsParticipants()) { String participantId = elasticSearchParticipantDto.getParticipantId(); elasticSearchParticipantDto.getDsm().ifPresent(esDsm -> participantDataByParticipantId.put(participantId, esDsm.getParticipantData())); @@ -221,7 +221,7 @@ private List collectData(DDPInstance ddpInstance) { ElasticSearch participantsByIds = elasticSearchable.getParticipantsByIds(usersIndexES, proxyGuids); List proxies = participantsByIds.getEsParticipants(); - List participantData = esDsm.getParticipantData(); + List participantData = esDsm.getParticipantData(); ParticipantWrapperDto participantWrapperDto = new ParticipantWrapperDto(); participantWrapperDto.setEsData(elasticSearchParticipantDto); @@ -251,7 +251,7 @@ private void mapTissueToProperOncHistoryDetail(List oncHistory } } - void sortBySelfElseById(Collection> participantDatas) { + void sortBySelfElseById(Collection> participantDatas) { participantDatas.forEach(pDataList -> pDataList.sort((o1, o2) -> { Map pData = new Gson().fromJson(o1.getData().orElse(""), new TypeToken>() {}.getType()); if (Objects.nonNull(pData) && FamilyMemberConstants.MEMBER_TYPE_SELF.equals(pData.get(FamilyMemberConstants.MEMBER_TYPE))) return -1; diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapperDto.java b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapperDto.java index 1050e1469..64bc0e3a6 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapperDto.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapperDto.java @@ -16,7 +16,7 @@ import org.broadinstitute.dsm.db.MedicalRecord; import org.broadinstitute.dsm.db.OncHistoryDetail; import org.broadinstitute.dsm.db.Participant; -import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; +import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,11 +35,11 @@ public class ParticipantWrapperDto { private List abstractionActivities; private List abstractionSummary; private List proxyData; - private List participantData; + private List participantData; public ParticipantWrapperDto(ElasticSearchParticipantDto esData, Participant participant, List medicalRecords, List oncHistoryDetails, List kits, List abstractionActivities, - List abstractionSummary, List proxyData, List participantData) { + List abstractionSummary, List proxyData, List participantData) { this.esData = esData; this.participant = participant; this.medicalRecords = medicalRecords; diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/data/AddFamilyMemberPayload.java b/src/main/java/org/broadinstitute/dsm/model/participant/data/AddFamilyMemberPayload.java index 03decd7ce..319fb2bcb 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/data/AddFamilyMemberPayload.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/data/AddFamilyMemberPayload.java @@ -9,9 +9,8 @@ import com.google.gson.Gson; import lombok.Setter; -import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; -import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; +import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData; import org.broadinstitute.dsm.model.bookmark.Bookmark; @Setter @@ -102,9 +101,9 @@ public String generateCollaboratorParticipantId() { this.data.getSubjectId(); } - public long getFamilyId(List participantDataDtos) throws NoSuchFieldException { + public long getFamilyId(List participantData) throws NoSuchFieldException { String familyId = null; - for (ParticipantDataDto pDataDto: Objects.requireNonNull(participantDataDtos)) { + for (ParticipantData pDataDto: Objects.requireNonNull(participantData)) { Map pDataMap = new Gson().fromJson(pDataDto.getData().orElse(""), Map.class); familyId = pDataMap.get(FamilyMemberConstants.FAMILY_ID); if (org.apache.commons.lang3.StringUtils.isNumeric(familyId)) break; @@ -114,8 +113,8 @@ public long getFamilyId(List participantDataDtos) throws NoS } public long getOrGenerateFamilyId() { - ParticipantData participantData = new ParticipantData(); - List participantDataByParticipantId = participantData.getParticipantDataByParticipantId(this.participantId); + org.broadinstitute.dsm.model.participant.data.ParticipantData participantData = new org.broadinstitute.dsm.model.participant.data.ParticipantData(); + List participantDataByParticipantId = participantData.getParticipantDataByParticipantId(this.participantId); long familyId; try { familyId = getFamilyId(participantDataByParticipantId); diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/data/FamilyMemberDetails.java b/src/main/java/org/broadinstitute/dsm/model/participant/data/FamilyMemberDetails.java index d3ad00a9d..2fdcc7c13 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/data/FamilyMemberDetails.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/data/FamilyMemberDetails.java @@ -9,12 +9,8 @@ import java.util.Objects; import java.util.stream.Collectors; -import com.google.gson.Gson; import com.google.gson.annotations.SerializedName; import lombok.Data; -import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; -import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; -import org.broadinstitute.dsm.model.bookmark.Bookmark; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import spark.utils.StringUtils; diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java b/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java index 4ec0056da..8e1c26ba5 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java @@ -16,7 +16,6 @@ import org.broadinstitute.dsm.db.dao.Dao; import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; -import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; import org.broadinstitute.dsm.model.elastic.ESProfile; import org.broadinstitute.dsm.util.ParticipantUtil; import org.slf4j.Logger; @@ -53,19 +52,19 @@ public ParticipantData(long participantDataId, String ddpParticipantId, int ddpI this.data = data; } - public static ParticipantData parseDto(@NonNull ParticipantDataDto participantDataDto) { + public static ParticipantData parseDto(@NonNull org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData participantData) { return new ParticipantData( - participantDataDto.getParticipantDataId(), - participantDataDto.getDdpParticipantId().orElse(""), - participantDataDto.getDdpInstanceId(), - participantDataDto.getFieldTypeId().orElse(""), - GSON.fromJson(participantDataDto.getData().orElse(""), new TypeToken>() {}.getType()) + participantData.getParticipantDataId(), + participantData.getDdpParticipantId().orElse(""), + participantData.getDdpInstanceId(), + participantData.getFieldTypeId().orElse(""), + GSON.fromJson(participantData.getData().orElse(""), new TypeToken>() {}.getType()) ); } - public static List parseDtoList(@NonNull List participantDataDtoList) { + public static List parseDtoList(@NonNull List participantDataList) { List participantData = new ArrayList<>(); - participantDataDtoList.forEach(dto -> participantData.add(new ParticipantData( + participantDataList.forEach(dto -> participantData.add(new ParticipantData( dto.getParticipantDataId(), dto.getDdpParticipantId().orElse(""), dto.getDdpInstanceId(), @@ -75,7 +74,7 @@ public static List parseDtoList(@NonNull List getParticipantDataByParticipantId(String ddpParticipantId) { + public List getParticipantDataByParticipantId(String ddpParticipantId) { if (StringUtils.isBlank(ddpParticipantId)) return Collections.emptyList(); ParticipantDataDao dataAccess = (ParticipantDataDao) setDataAccess(new ParticipantDataDao()); return dataAccess.getParticipantDataByParticipantId(ddpParticipantId); @@ -101,8 +100,8 @@ public void setData(String ddpParticipantId, int ddpInstanceId, String fieldType public long insertParticipantData(String userEmail) { dataAccess = new ParticipantDataDao(); - ParticipantDataDto participantDataDto = - new ParticipantDataDto.Builder() + org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData participantData = + new org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData.Builder() .withDdpParticipantId(this.ddpParticipantId) .withDdpInstanceId(this.ddpInstanceId) .withFieldTypeId(this.fieldTypeId) @@ -113,7 +112,7 @@ public long insertParticipantData(String userEmail) { if (isRelationshipIdExists()) { throw new RuntimeException(String.format("Family member with that Relationship ID: %s already exists", getRelationshipId())); } - int createdDataKey = dataAccess.create(participantDataDto); + int createdDataKey = dataAccess.create(participantData); if (createdDataKey < 1) { throw new RuntimeException("Could not insert participant data for : " + this.ddpParticipantId); } @@ -141,7 +140,7 @@ String getRelationshipId() { } public boolean updateParticipantData(int dataId, String changedByUser) { - ParticipantDataDto participantDataDto = new ParticipantDataDto.Builder() + org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData participantData = new org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData.Builder() .withParticipantDataId(dataId) .withDdpParticipantId(this.ddpParticipantId) .withDdpInstanceId(this.ddpInstanceId) @@ -150,12 +149,12 @@ public boolean updateParticipantData(int dataId, String changedByUser) { .withLastChanged(System.currentTimeMillis()) .withChangedBy(changedByUser) .build(); - int rowsAffected = ((ParticipantDataDao) dataAccess).updateParticipantDataColumn(participantDataDto); + int rowsAffected = ((ParticipantDataDao) dataAccess).updateParticipantDataColumn(participantData); return rowsAffected == 1; } - public Optional findProband(List participantDataDtoList) { - return Objects.requireNonNull(participantDataDtoList).stream() + public Optional findProband(List participantDataList) { + return Objects.requireNonNull(participantDataList).stream() .filter(participantDataDto -> { Map pDataMap = GSON.fromJson(participantDataDto.getData().orElse(""), Map.class); return FamilyMemberConstants.MEMBER_TYPE_SELF.equals(pDataMap.get(FamilyMemberConstants.MEMBER_TYPE)); diff --git a/src/main/java/org/broadinstitute/dsm/pubsub/WorkflowStatusUpdate.java b/src/main/java/org/broadinstitute/dsm/pubsub/WorkflowStatusUpdate.java index c5cb99104..a83fbdf4f 100644 --- a/src/main/java/org/broadinstitute/dsm/pubsub/WorkflowStatusUpdate.java +++ b/src/main/java/org/broadinstitute/dsm/pubsub/WorkflowStatusUpdate.java @@ -11,7 +11,7 @@ import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; -import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; +import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData; import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,7 +41,7 @@ public static void updateCustomWorkflow(Map attributesMap, Strin String ddpParticipantId = attributesMap.get(PARTICIPANT_GUID); DDPInstance instance = DDPInstance.getDDPInstanceByGuid(studyGuid); - List participantDatas = participantDataDao.getParticipantDataByParticipantId(ddpParticipantId); + List participantDatas = participantDataDao.getParticipantDataByParticipantId(ddpParticipantId); Optional fieldSetting = fieldSettingsDao.getFieldSettingByColumnNameAndInstanceId(Integer.parseInt(instance.getDdpInstanceId()), workflow); if (fieldSetting.isEmpty()) { logger.warn("Wrong workflow name"); @@ -63,7 +63,7 @@ public static void updateCustomWorkflow(Map attributesMap, Strin } public static void exportToESifNecessary(String workflow, String status, String ddpParticipantId, - DDPInstance instance, FieldSettingsDto setting, List participantDatas) { + DDPInstance instance, FieldSettingsDto setting, List participantDatas) { String actions = setting.getActions(); if (actions == null) { return; @@ -83,8 +83,8 @@ public static void exportToESifNecessary(String workflow, String status, String } } - private static Optional getProbandStudySpecificData(List participantDatas) { - for (ParticipantDataDto participantData: participantDatas) { + private static Optional getProbandStudySpecificData(List participantDatas) { + for (ParticipantData participantData: participantDatas) { String data = participantData.getData().orElse(null); if (data == null) { continue; @@ -107,7 +107,7 @@ public static int addNewParticipantDataWithStatus(String workflow, String status dataJsonObject.addProperty(workflow, status); int participantDataId; participantDataId = participantDataDao.create( - new ParticipantDataDto.Builder() + new ParticipantData.Builder() .withDdpParticipantId(ddpParticipantId) .withDdpInstanceId(setting.getDdpInstanceId()) .withFieldTypeId(setting.getFieldType()) @@ -119,20 +119,20 @@ public static int addNewParticipantDataWithStatus(String workflow, String status return participantDataId; } - public static void updateProbandStatusInDB(String workflow, String status, ParticipantDataDto participantDataDto, String studyGuid) { - String oldData = participantDataDto.getData().orElse(null); + public static void updateProbandStatusInDB(String workflow, String status, ParticipantData participantData, String studyGuid) { + String oldData = participantData.getData().orElse(null); if (oldData == null) { return; } JsonObject dataJsonObject = gson.fromJson(oldData, JsonObject.class); - if ((participantDataDto.getFieldTypeId().orElse("").contains("GROUP") || isProband(gson.fromJson(dataJsonObject, Map.class)))) { + if ((participantData.getFieldTypeId().orElse("").contains("GROUP") || isProband(gson.fromJson(dataJsonObject, Map.class)))) { dataJsonObject.addProperty(workflow, status); participantDataDao.updateParticipantDataColumn( - new ParticipantDataDto.Builder() - .withParticipantDataId(participantDataDto.getParticipantDataId()) - .withDdpParticipantId(participantDataDto.getDdpParticipantId().orElse("")) - .withDdpInstanceId(participantDataDto.getDdpInstanceId()) - .withFieldTypeId(participantDataDto.getFieldTypeId().orElse("")) + new ParticipantData.Builder() + .withParticipantDataId(participantData.getParticipantDataId()) + .withDdpParticipantId(participantData.getDdpParticipantId().orElse("")) + .withDdpInstanceId(participantData.getDdpInstanceId()) + .withFieldTypeId(participantData.getFieldTypeId().orElse("")) .withData(dataJsonObject.toString()) .withLastChanged(System.currentTimeMillis()) .withChangedBy(DSS) diff --git a/src/main/java/org/broadinstitute/dsm/util/ParticipantUtil.java b/src/main/java/org/broadinstitute/dsm/util/ParticipantUtil.java index d0392a01a..d21199027 100644 --- a/src/main/java/org/broadinstitute/dsm/util/ParticipantUtil.java +++ b/src/main/java/org/broadinstitute/dsm/util/ParticipantUtil.java @@ -3,7 +3,7 @@ import com.google.gson.Gson; import lombok.NonNull; import org.apache.commons.lang3.StringUtils; -import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; +import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData; import org.broadinstitute.dsm.model.elastic.ESProfile; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; @@ -48,9 +48,9 @@ public static boolean matchesApplicantEmail(ESProfile applicantProfile, return applicantEmail.equalsIgnoreCase(currentParticipantEmail); } - public static boolean matchesApplicantEmail(String collaboratorParticipantId, List participantDatas) { + public static boolean matchesApplicantEmail(String collaboratorParticipantId, List participantDatas) { String applicantEmail = null, currentParticipantEmail = null; - for (ParticipantDataDto participantData: participantDatas) { + for (ParticipantData participantData: participantDatas) { Map dataMap = participantData.getDataMap(); if (dataMap == null) { continue; @@ -91,10 +91,10 @@ public static String getParticipantEmailById(String esParticipantIndex, String p return email.toString(); } - public static ParticipantDataDto findApplicantData(String ddpParticipantId, List participantDataDtos) { - ParticipantDataDto applicantData = null; - for (ParticipantDataDto participantDataDto : participantDataDtos) { - Map dataMap = participantDataDto.getDataMap(); + public static ParticipantData findApplicantData(String ddpParticipantId, List participantData) { + ParticipantData applicantData = null; + for (ParticipantData participantData : participantData) { + Map dataMap = participantData.getDataMap(); if (dataMap == null) { continue; } @@ -103,7 +103,7 @@ public static ParticipantDataDto findApplicantData(String ddpParticipantId, List boolean isNewApplicant = dataMap.containsKey(FamilyMemberConstants.IS_APPLICANT) && TRUE.equals(dataMap.get(FamilyMemberConstants.IS_APPLICANT)); if (isOldApplicant || isNewApplicant) { - applicantData = participantDataDto; + applicantData = participantData; break; } } diff --git a/src/test/java/org/broadinstitute/dsm/ParticipantTest.java b/src/test/java/org/broadinstitute/dsm/ParticipantTest.java index 477e17f80..64f326a5b 100644 --- a/src/test/java/org/broadinstitute/dsm/ParticipantTest.java +++ b/src/test/java/org/broadinstitute/dsm/ParticipantTest.java @@ -1,6 +1,6 @@ package org.broadinstitute.dsm; -import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; +import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData; import org.broadinstitute.dsm.util.ParticipantUtil; import org.junit.Assert; import org.junit.Before; @@ -10,13 +10,13 @@ public class ParticipantTest { - List participantDatas; - List newParticipantDatas; + List participantDatas; + List newParticipantDatas; @Before public void initializeParticipantDatas() { participantDatas = List.of( - new ParticipantDataDto.Builder() + new ParticipantData.Builder() .withDdpParticipantId("testId") .withDdpInstanceId(19) .withFieldTypeId("testFieldType") @@ -24,7 +24,7 @@ public void initializeParticipantDatas() { .withLastChanged(0) .withChangedBy(null) .build(), - new ParticipantDataDto.Builder() + new ParticipantData.Builder() .withDdpParticipantId("testId2") .withDdpInstanceId(19) .withFieldTypeId("testFieldType") @@ -32,7 +32,7 @@ public void initializeParticipantDatas() { .withLastChanged(0) .withChangedBy(null) .build(), - new ParticipantDataDto.Builder() + new ParticipantData.Builder() .withDdpParticipantId("testId3") .withDdpInstanceId(19) .withFieldTypeId("testFieldType") @@ -43,7 +43,7 @@ public void initializeParticipantDatas() { ); newParticipantDatas = List.of( - new ParticipantDataDto.Builder() + new ParticipantData.Builder() .withDdpParticipantId("testId") .withDdpInstanceId(19) .withFieldTypeId("testFieldType") @@ -51,7 +51,7 @@ public void initializeParticipantDatas() { .withLastChanged(0) .withChangedBy(null) .build(), - new ParticipantDataDto.Builder() + new ParticipantData.Builder() .withDdpParticipantId("testId2") .withDdpInstanceId(19) .withFieldTypeId("testFieldType") @@ -59,7 +59,7 @@ public void initializeParticipantDatas() { .withLastChanged(0) .withChangedBy(null) .build(), - new ParticipantDataDto.Builder() + new ParticipantData.Builder() .withDdpParticipantId("testId3") .withDdpInstanceId(19) .withFieldTypeId("testFieldType") diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java index c6ee77072..da63f3a84 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java @@ -5,13 +5,11 @@ import java.lang.reflect.Field; import java.util.*; -import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.Participant; import org.broadinstitute.dsm.db.dao.settings.FieldSettingsDao; -import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; +import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData; import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; import org.broadinstitute.dsm.model.FollowUp; -import org.broadinstitute.dsm.model.elastic.export.generate.SourceGeneratorTest; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -82,7 +80,7 @@ public void transformJsonToMap() { String json = "{\"DDP_INSTANCE\": \"TEST\", \"DDP_VALUE\": \"VALUE\", \"BOOLEAN_VAL\": \"true\", \"LONG_VAL\": \"5\"}"; - ParticipantDataDto participantDataDto = new ParticipantDataDto.Builder() + ParticipantData participantData = new ParticipantData.Builder() .withParticipantDataId(10) .withDdpParticipantId("123") .withDdpInstanceId(55) @@ -90,7 +88,7 @@ public void transformJsonToMap() { .withData(json) .build(); - Map result = Util.transformObjectToMap(participantDataDto, "angio"); + Map result = Util.transformObjectToMap(participantData, "angio"); assertEquals("TEST", ((Map) result.get("dynamicFields")).get("ddpInstance")); assertEquals("VALUE", ((Map) result.get("dynamicFields")).get("ddpValue")); assertEquals(true, ((Map) result.get("dynamicFields")).get("booleanVal")); diff --git a/src/test/java/org/broadinstitute/dsm/model/participant/ParticipantWrapperTest.java b/src/test/java/org/broadinstitute/dsm/model/participant/ParticipantWrapperTest.java index f264e7adb..afcc08323 100644 --- a/src/test/java/org/broadinstitute/dsm/model/participant/ParticipantWrapperTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/participant/ParticipantWrapperTest.java @@ -9,7 +9,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; +import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData; import org.broadinstitute.dsm.model.elastic.ESProfile; import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; @@ -71,7 +71,7 @@ public void sortBySelfElseById() { Random random = new Random(); String[] memberTypes = new String[] {"SISTER", "COUSIN", "SELF", "BROTHER"}; AtomicInteger i = new AtomicInteger(0); - List pDatas = Stream.generate(() -> new ParticipantDataDto.Builder() + List pDatas = Stream.generate(() -> new ParticipantData.Builder() .withData(String.format("{\"MEMTER_TYPE\":\"%s\"}", memberTypes[i.getAndIncrement()])) .withParticipantDataId(random.nextInt(100)) .build()).limit(4).collect(Collectors.toList()); diff --git a/src/test/java/org/broadinstitute/dsm/model/participant/data/ParticipantDataTest.java b/src/test/java/org/broadinstitute/dsm/model/participant/data/ParticipantDataTest.java index 3b70148b5..4d6828d65 100644 --- a/src/test/java/org/broadinstitute/dsm/model/participant/data/ParticipantDataTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/participant/data/ParticipantDataTest.java @@ -7,7 +7,7 @@ import java.util.Random; import com.google.gson.Gson; -import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; +import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -16,30 +16,30 @@ public class ParticipantDataTest { private static final String[] MEMBER_TYPES = {"SELF", "SISTER", "SON", "MOTHER", "FATHER", "COUSIN"}; private static final Gson GSON = new Gson(); - private static ParticipantData participantData; + private static org.broadinstitute.dsm.model.participant.data.ParticipantData participantData; @BeforeClass public static void setUp() { - participantData = new ParticipantData(); + participantData = new org.broadinstitute.dsm.model.participant.data.ParticipantData(); } @Test public void testFindProband() { - List participantDataDtoList = generateParticipantData(); - Optional maybeProbandData = participantData.findProband(participantDataDtoList); - Map map = GSON.fromJson(maybeProbandData.flatMap(ParticipantDataDto::getData).orElse(""), Map.class); + List participantDataList = generateParticipantData(); + Optional maybeProbandData = participantData.findProband(participantDataList); + Map map = GSON.fromJson(maybeProbandData.flatMap(ParticipantData::getData).orElse(""), Map.class); Assert.assertEquals("SELF", map.get("MEMBER_TYPE")); } @Test public void testGetFamilyId() { - List participantDataDtoList = generateParticipantData(); - ParticipantDataDto maybeProbandData = participantData.findProband(participantDataDtoList).get(); + List participantDataList = generateParticipantData(); + ParticipantData maybeProbandData = participantData.findProband(participantDataList).get(); maybeProbandData.setData("{\"FAMILY_ID\":\"2005\"}"); AddFamilyMemberPayload addFamilyMemberPayload = new AddFamilyMemberPayload.Builder("", "").build(); long familyId = 0; try { - familyId = addFamilyMemberPayload.getFamilyId(participantDataDtoList); + familyId = addFamilyMemberPayload.getFamilyId(participantDataList); } catch (NoSuchFieldException e) { Assert.fail(); } @@ -48,13 +48,13 @@ public void testGetFamilyId() { @Test public void testFamilyMemberHasNotApplicantEmail() { - ParticipantData participantData = new ParticipantData(0, "", 0, "", Map.of(FamilyMemberConstants.EMAIL, "familymember@mail.com")); + org.broadinstitute.dsm.model.participant.data.ParticipantData participantData = new org.broadinstitute.dsm.model.participant.data.ParticipantData(0, "", 0, "", Map.of(FamilyMemberConstants.EMAIL, "familymember@mail.com")); Assert.assertFalse(participantData.hasFamilyMemberApplicantEmail()); } - private List generateParticipantData() { + private List generateParticipantData() { Random random = new Random(); - List participantDataDtoList = new ArrayList<>(); + List participantDataList = new ArrayList<>(); for (int i = 0; i < MEMBER_TYPES.length; i++) { String memberType = MEMBER_TYPES[i]; int randomGeneratedFamilyId = random.nextInt(); @@ -69,8 +69,8 @@ private List generateParticipantData() { collaboratorParticipantId); familyMemberDetails.setEmail(email); String data = GSON.toJson(familyMemberDetails); - ParticipantDataDto participantDataDto = - new ParticipantDataDto.Builder() + ParticipantData participantData = + new ParticipantData.Builder() .withDdpParticipantId(collaboratorParticipantId) .withDdpInstanceId(i) .withFieldTypeId("") @@ -78,9 +78,9 @@ private List generateParticipantData() { .withLastChanged(System.currentTimeMillis()) .withChangedBy("SYSTEM") .build(); - participantDataDtoList.add(participantDataDto); + participantDataList.add(participantData); } - return participantDataDtoList; + return participantDataList; } diff --git a/src/test/java/org/broadinstitute/dsm/pubsub/UpdateWorkflowStatusTest.java b/src/test/java/org/broadinstitute/dsm/pubsub/UpdateWorkflowStatusTest.java index faa37f884..5501a8408 100644 --- a/src/test/java/org/broadinstitute/dsm/pubsub/UpdateWorkflowStatusTest.java +++ b/src/test/java/org/broadinstitute/dsm/pubsub/UpdateWorkflowStatusTest.java @@ -9,7 +9,7 @@ import org.broadinstitute.dsm.db.dao.user.UserDao; import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; -import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; +import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData; import org.broadinstitute.dsm.util.DBTestUtil; import org.junit.AfterClass; import org.junit.Assert; @@ -35,7 +35,7 @@ public class UpdateWorkflowStatusTest { private static int participantDataId; - private static ParticipantDataDto participantDataDto; + private static ParticipantData participantData; private static final Map participantData = new HashMap<>(); @@ -66,7 +66,7 @@ public static void doFirst() { public void testUpdateProbandStatusInDB() { String workflow = "REGISTRATION_STATUS"; String status = "ENROLLED"; - WorkflowStatusUpdate.updateProbandStatusInDB(workflow, status, participantDataDto, RGP); + WorkflowStatusUpdate.updateProbandStatusInDB(workflow, status, participantData, RGP); String data = participantDataDao.get(participantDataId).orElseThrow().getData().orElse(""); JsonObject dataJsonObject = gson.fromJson(data, JsonObject.class); Assert.assertEquals(status, dataJsonObject.get(workflow).getAsString()); @@ -94,8 +94,8 @@ private static void createDataForParticipant() { } private static void createParticipantData() { - participantDataDto = - new ParticipantDataDto.Builder() + participantData = + new ParticipantData.Builder() .withDdpParticipantId(participantId) .withDdpInstanceId(ddpInstanceDto.getDdpInstanceId()) .withFieldTypeId(UPDATE_WORKFLOW_TEST) @@ -103,9 +103,9 @@ private static void createParticipantData() { .withLastChanged(System.currentTimeMillis()) .withChangedBy(userDto.getEmail().orElse("")) .build(); - participantDataId = participantDataDao.create(participantDataDto); - participantDataDto = - new ParticipantDataDto.Builder() + participantDataId = participantDataDao.create(participantData); + participantData = + new ParticipantData.Builder() .withParticipantDataId(participantDataId) .withDdpParticipantId(participantId) .withDdpInstanceId(ddpInstanceDto.getDdpInstanceId()) diff --git a/src/test/java/org/broadinstitute/dsm/route/familymember/AddFamilyMemberRouteTest.java b/src/test/java/org/broadinstitute/dsm/route/familymember/AddFamilyMemberRouteTest.java index e5d672374..54f101560 100644 --- a/src/test/java/org/broadinstitute/dsm/route/familymember/AddFamilyMemberRouteTest.java +++ b/src/test/java/org/broadinstitute/dsm/route/familymember/AddFamilyMemberRouteTest.java @@ -14,10 +14,9 @@ import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; import org.broadinstitute.dsm.db.dao.user.UserDao; import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; -import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantDataDto; +import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData; import org.broadinstitute.dsm.model.participant.data.AddFamilyMemberPayload; import org.broadinstitute.dsm.model.participant.data.FamilyMemberDetails; -import org.broadinstitute.dsm.model.participant.data.ParticipantData; import org.broadinstitute.dsm.util.DBTestUtil; import org.junit.AfterClass; import org.junit.Assert; @@ -76,16 +75,16 @@ private static void createProbandTestData() { } private static void createProbandTestParticipantData() { - ParticipantDataDto probandParticipantDataDto = - new ParticipantDataDto.Builder() + ParticipantData probandParticipantData = + new ParticipantData.Builder() .withDdpParticipantId(participantId) .withDdpInstanceId(ddpInstanceDto.getDdpInstanceId()) - .withFieldTypeId(ParticipantData.FIELD_TYPE_PARTICIPANTS) + .withFieldTypeId(org.broadinstitute.dsm.model.participant.data.ParticipantData.FIELD_TYPE_PARTICIPANTS) .withData(gson.toJson(probandData)) .withLastChanged(System.currentTimeMillis()) .withChangedBy(userDto.getEmail().orElse("SYSTEM")) .build(); - ddpExistingProbandParticipantDataId = participantDataDao.create(probandParticipantDataDto); + ddpExistingProbandParticipantDataId = participantDataDao.create(probandParticipantData); } @@ -153,9 +152,9 @@ public void noUserIdProvided() { public void relationshipIdAlreadyExists() { String payload = payloadFactory(participantId, ddpInstanceDto.getInstanceName(), probandData, userDto.getId()); AddFamilyMemberPayload addFamilyMemberPayload = gson.fromJson(payload, AddFamilyMemberPayload.class); - ParticipantData participantData = new ParticipantData(participantDataDao); + org.broadinstitute.dsm.model.participant.data.ParticipantData participantData = new org.broadinstitute.dsm.model.participant.data.ParticipantData(participantDataDao); participantData.setData(addFamilyMemberPayload.getParticipantId().get(), ddpInstanceDto.getDdpInstanceId(), - ddpInstanceDto.getInstanceName() + ParticipantData.FIELD_TYPE_PARTICIPANTS, probandData); + ddpInstanceDto.getInstanceName() + org.broadinstitute.dsm.model.participant.data.ParticipantData.FIELD_TYPE_PARTICIPANTS, probandData); Assert.assertTrue(participantData.isRelationshipIdExists()); } @@ -165,16 +164,16 @@ public void addFamilyMemberToParticipant() { Result result = new Result(200); AddFamilyMemberPayload addFamilyMemberPayload = gson.fromJson(payload, AddFamilyMemberPayload.class); try { - ParticipantDataDto participantDataDto = - new ParticipantDataDto.Builder() + ParticipantData participantData = + new ParticipantData.Builder() .withDdpParticipantId(addFamilyMemberPayload.getParticipantId().get()) .withDdpInstanceId(ddpInstanceDto.getDdpInstanceId()) - .withFieldTypeId(ddpInstanceDto.getInstanceName() + ParticipantData.FIELD_TYPE_PARTICIPANTS) + .withFieldTypeId(ddpInstanceDto.getInstanceName() + org.broadinstitute.dsm.model.participant.data.ParticipantData.FIELD_TYPE_PARTICIPANTS) .withData(gson.toJson(addFamilyMemberPayload.getData().get())) .withLastChanged(System.currentTimeMillis()) .withChangedBy(userDto.getEmail().orElse("SYSTEM")) .build(); - ddpFamilyMemberParticipantDataId = participantDataDao.create(participantDataDto); + ddpFamilyMemberParticipantDataId = participantDataDao.create(participantData); } catch (Exception e) { result = new Result(500); } From b8c2184e56086c74f065ca9332b986588b9ddf0a Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 26 Jan 2022 16:11:06 +0400 Subject: [PATCH 347/405] DDP-7443 | change constructors from default values to null, add setRefresh to updateByQuery to avoid conflicts --- .../broadinstitute/dsm/db/KitRequestShipping.java | 12 ++++++------ .../elastic/export/painless/UpsertPainless.java | 2 ++ .../org/broadinstitute/dsm/util/ParticipantUtil.java | 4 ++-- .../dsm/pubsub/UpdateWorkflowStatusTest.java | 10 +++++----- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java index eec487add..c78c76073 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java @@ -258,15 +258,15 @@ public KitRequestShipping(String collaboratorParticipantId, String kitTypeName, String upsTrackingStatus, String upsReturnStatus, String externalOrderStatus, String externalOrderNumber, Long externalOrderDate, Boolean careEvolve, String uploadReason) { this(null, collaboratorParticipantId, null, null, null, kitTypeName, dsmKitRequestId, null, null, null, null, null, null, null, scanDate, error, null, receiveDate, - null, deactivatedDate, null, null, false, null, null, null, externalOrderNumber, false, externalOrderStatus, null, + null, deactivatedDate, null, null, null, null, null, null, externalOrderNumber, null, externalOrderStatus, null, testResult, - upsTrackingStatus, upsReturnStatus, externalOrderDate, false, uploadReason, null, null, null); + upsTrackingStatus, upsReturnStatus, externalOrderDate, careEvolve, uploadReason, null, null, null); } public KitRequestShipping(String participantId, String collaboratorParticipantId, String dsmKitId, String realm, String trackingNumberTo, String receiveDateString, String hruid, String gender) { this(participantId, collaboratorParticipantId, null, null, realm, null, null, null, null, null, - trackingNumberTo, null, null, null, null, false, null, null, - null, null, null, dsmKitId, false, null, null, null, null, false, null, null, null, null, null, null, false, null, + trackingNumberTo, null, null, null, null, null, null, null, + null, null, null, dsmKitId, null, null, null, null, null, null, null, null, null, null, null, null, null, null, receiveDateString, hruid, gender); } @@ -274,8 +274,8 @@ public KitRequestShipping(Long dsmKitRequestId, Long dsmKitId, String easypostTo String message) { this(null, null, null, null, null, null, dsmKitRequestId, dsmKitId, null, null, null, null, null, null, null, error, message, null, - easypostAddressId, null, null, null, false, easypostToId, null, null, null, false, null, null, null, null, null, null, - false, + easypostAddressId, null, null, null, null, easypostToId, null, null, null, null, null, null, null, null, null, null, + null, null, null, null, null); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/UpsertPainless.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/UpsertPainless.java index 437bcbeb3..784a5befc 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/UpsertPainless.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/UpsertPainless.java @@ -34,6 +34,8 @@ public void export() { UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(index); updateByQueryRequest.setQuery(queryBuilder); updateByQueryRequest.setScript(painless); + updateByQueryRequest.setMaxRetries(5); + updateByQueryRequest.setRefresh(true); try { clientInstance.updateByQuery(updateByQueryRequest, RequestOptions.DEFAULT); } catch (IOException e) { diff --git a/src/main/java/org/broadinstitute/dsm/util/ParticipantUtil.java b/src/main/java/org/broadinstitute/dsm/util/ParticipantUtil.java index d21199027..597963912 100644 --- a/src/main/java/org/broadinstitute/dsm/util/ParticipantUtil.java +++ b/src/main/java/org/broadinstitute/dsm/util/ParticipantUtil.java @@ -91,9 +91,9 @@ public static String getParticipantEmailById(String esParticipantIndex, String p return email.toString(); } - public static ParticipantData findApplicantData(String ddpParticipantId, List participantData) { + public static ParticipantData findApplicantData(String ddpParticipantId, List participantsDatas) { ParticipantData applicantData = null; - for (ParticipantData participantData : participantData) { + for (ParticipantData participantData : participantsDatas) { Map dataMap = participantData.getDataMap(); if (dataMap == null) { continue; diff --git a/src/test/java/org/broadinstitute/dsm/pubsub/UpdateWorkflowStatusTest.java b/src/test/java/org/broadinstitute/dsm/pubsub/UpdateWorkflowStatusTest.java index 5501a8408..01ca85c8a 100644 --- a/src/test/java/org/broadinstitute/dsm/pubsub/UpdateWorkflowStatusTest.java +++ b/src/test/java/org/broadinstitute/dsm/pubsub/UpdateWorkflowStatusTest.java @@ -37,7 +37,7 @@ public class UpdateWorkflowStatusTest { private static ParticipantData participantData; - private static final Map participantData = new HashMap<>(); + private static final Map testParticipantData = new HashMap<>(); private static DDPInstanceDto ddpInstanceDto; private static final DDPInstanceDao ddpInstanceDao = new DDPInstanceDao(); @@ -89,8 +89,8 @@ public void testAddNewParticipantDataWithStatus() { } private static void createDataForParticipant() { - participantData.put("REGISTRATION_STATUS", "REGISTERED"); - participantData.put("MEMBER_TYPE", "SELF"); + testParticipantData.put("REGISTRATION_STATUS", "REGISTERED"); + testParticipantData.put("MEMBER_TYPE", "SELF"); } private static void createParticipantData() { @@ -99,7 +99,7 @@ private static void createParticipantData() { .withDdpParticipantId(participantId) .withDdpInstanceId(ddpInstanceDto.getDdpInstanceId()) .withFieldTypeId(UPDATE_WORKFLOW_TEST) - .withData(gson.toJson(participantData)) + .withData(gson.toJson(testParticipantData)) .withLastChanged(System.currentTimeMillis()) .withChangedBy(userDto.getEmail().orElse("")) .build(); @@ -110,7 +110,7 @@ private static void createParticipantData() { .withDdpParticipantId(participantId) .withDdpInstanceId(ddpInstanceDto.getDdpInstanceId()) .withFieldTypeId(UPDATE_WORKFLOW_TEST) - .withData(gson.toJson(participantData)) + .withData(gson.toJson(testParticipantData)) .withLastChanged(System.currentTimeMillis()) .withChangedBy(userDto.getEmail().orElse("")) .build(); From 6c2bd97ad26bd0a97a8f91f57e9b4dd332413256 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 27 Jan 2022 11:01:03 +0400 Subject: [PATCH 348/405] DDP-7121 | add IS NULL operator and spliter --- .../org/broadinstitute/dsm/model/Filter.java | 1 + .../dsm/model/elastic/filter/Operator.java | 17 +++++--------- .../filter/query/QueryBuilderFactory.java | 23 +++++++++++++++++-- .../filter/splitter/IsNullSplitter.java | 11 +++++++++ .../filter/splitter/SplitterFactory.java | 7 +++--- 5 files changed, 42 insertions(+), 17 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/IsNullSplitter.java diff --git a/src/main/java/org/broadinstitute/dsm/model/Filter.java b/src/main/java/org/broadinstitute/dsm/model/Filter.java index b229bb23c..ab4de3ae3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/Filter.java +++ b/src/main/java/org/broadinstitute/dsm/model/Filter.java @@ -63,6 +63,7 @@ public class Filter { public static final String DATE_LESS = SMALLER_EQUALS + DATE_FORMAT; public static final char OPEN_PARENTHESIS_CHAR = '('; public static final char CLOSE_PARENTHESIS_CHAR = ')'; + public static final String IS_NULL_TRIMMED = "IS NULL"; public static String TEXT = "TEXT"; public static String OPTIONS = "OPTIONS"; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java index d8fa4d926..ea172ba5d 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java @@ -14,6 +14,7 @@ public enum Operator { GREATER_THAN_EQUALS(Filter.LARGER_EQUALS_TRIMMED), LESS_THAN_EQUALS(Filter.SMALLER_EQUALS_TRIMMED), IS_NOT_NULL(Filter.IS_NOT_NULL_TRIMMED), + IS_NULL(Filter.IS_NULL_TRIMMED), DIAMOND_EQUALS(Filter.DIAMOND_EQUALS), MULTIPLE_OPTIONS(Operator.MULTIPLE_OPTIONS_INDICATOR), STR_DATE(Filter.DATE_FORMAT), @@ -24,6 +25,8 @@ public enum Operator { public static final String MULTIPLE_OPTIONS_INDICATOR = "()"; public static final String UNKNOWN_OPERATOR = "Unknown operator"; + public static final List IS_NOT_NULL_LIST = Arrays.asList("IS", "NOT", + "NULL"); private String value; @@ -42,12 +45,10 @@ public static Operator extract(String filter) { String[] splittedFilter = filter.split(Filter.SPACE); if (isMultipleOptions(splittedFilter)) return MULTIPLE_OPTIONS; - else if (isNotNull(splittedFilter)) - return IS_NOT_NULL; Optional maybeOperator = Arrays.stream(splittedFilter) .filter(StringUtils::isNotBlank) .map(Operator::handleSpecialCaseOperators) - .filter(word -> Arrays.stream(Operator.values()).anyMatch(op -> op.value.equals(word))) + .filter(word -> Arrays.stream(Operator.values()).anyMatch(op -> op.value.equals(word)) || Operator.IS_NOT_NULL_LIST.contains(word)) .distinct() .reduce((prev, curr) -> String.join(Filter.SPACE, prev, curr)); if (maybeOperator.isPresent()) { @@ -68,6 +69,7 @@ else if (isNotNull(splittedFilter)) case "JSON_EXTRACT <=": case "JSON_EXTRACT LIKE": case "JSON_EXTRACT IS NOT NULL": + case "JSON_EXTRACT IS NULL": return JSON_EXTRACT; default: return Operator.getOperator(operator); @@ -81,7 +83,7 @@ private static String handleSpecialCaseOperators(String word) { String strOperator = StringUtils.EMPTY; for (Operator operator : Operator.values()) { int startIndex = word.indexOf(operator.value); - if (startIndex == -1) continue; + if (startIndex == -1 && !Operator.IS_NOT_NULL_LIST.contains(word)) continue; if (word.contains(Filter.OPEN_PARENTHESIS)) strOperator = word.substring(startIndex, startIndex + operator.value.length()); else strOperator = word; return strOperator; @@ -105,11 +107,4 @@ private static String[] cleanFromEmptySpaces(String[] splittedFilter) { .collect(Collectors.toList()) .toArray(new String[] {}); } - - private static boolean isNotNull(String[] splittedFilter) { - splittedFilter = cleanFromEmptySpaces(splittedFilter); - final int BOTTOM_SIZE_OF_IS_NOT_NULL = 4; - if (splittedFilter.length < BOTTOM_SIZE_OF_IS_NOT_NULL) return false; - return Filter.IS.equals(splittedFilter[1]) && Filter.NOT.equals(splittedFilter[2]) && Filter.NULL.equals(splittedFilter[3]); - } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryBuilderFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryBuilderFactory.java index 53324b6e3..ea54de7de 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryBuilderFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryBuilderFactory.java @@ -4,6 +4,7 @@ import org.broadinstitute.dsm.model.elastic.filter.Operator; import org.broadinstitute.dsm.model.elastic.filter.splitter.BaseSplitter; import org.broadinstitute.dsm.model.elastic.filter.splitter.GreaterThanEqualsSplitter; +import org.broadinstitute.dsm.model.elastic.filter.splitter.IsNullSplitter; import org.broadinstitute.dsm.model.elastic.filter.splitter.JsonExtractSplitter; import org.broadinstitute.dsm.model.elastic.filter.splitter.LessThanEqualsSplitter; import org.elasticsearch.index.query.BoolQueryBuilder; @@ -11,6 +12,7 @@ import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.index.query.WildcardQueryBuilder; public class QueryBuilderFactory { public static QueryBuilder buildQueryBuilder(Operator operator, QueryPayload payload, @@ -47,6 +49,9 @@ public static QueryBuilder buildQueryBuilder(Operator operator, QueryPayload pay case IS_NOT_NULL: qb = new ExistsQueryBuilder(payload.getFieldName()); break; + case IS_NULL: + qb = buildIsNullQuery(payload); + break; case MULTIPLE_OPTIONS: BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); Object[] values = payload.getValues(); @@ -57,8 +62,8 @@ public static QueryBuilder buildQueryBuilder(Operator operator, QueryPayload pay break; case JSON_EXTRACT: Object[] dynamicFieldValues = payload.getValues(); + JsonExtractSplitter jsonExtractSplitter = (JsonExtractSplitter) splitter; if (!StringUtils.EMPTY.equals(dynamicFieldValues[0])) { - JsonExtractSplitter jsonExtractSplitter = (JsonExtractSplitter) splitter; if (jsonExtractSplitter.getDecoratedSplitter() instanceof GreaterThanEqualsSplitter) { qb = new RangeQueryBuilder(payload.getFieldName()); ((RangeQueryBuilder)qb).gte(dynamicFieldValues[0]); @@ -69,7 +74,11 @@ public static QueryBuilder buildQueryBuilder(Operator operator, QueryPayload pay qb = new MatchQueryBuilder(payload.getFieldName(), dynamicFieldValues[0]); } } else { - qb = new ExistsQueryBuilder(payload.getFieldName()); + if (jsonExtractSplitter.getDecoratedSplitter() instanceof IsNullSplitter) { + qb = buildIsNullQuery(payload); + } else { + qb = new ExistsQueryBuilder(payload.getFieldName()); + } } break; default: @@ -77,4 +86,14 @@ public static QueryBuilder buildQueryBuilder(Operator operator, QueryPayload pay } return qb; } + + private static QueryBuilder buildIsNullQuery(QueryPayload payload) { + BoolQueryBuilder isNullQuery = new BoolQueryBuilder(); + BoolQueryBuilder existsWithEmpty = new BoolQueryBuilder(); + existsWithEmpty.must(new ExistsQueryBuilder(payload.getFieldName())); + existsWithEmpty.mustNot(new WildcardQueryBuilder(payload.getFieldName(), String.valueOf(payload.getValues()[0]))); + isNullQuery.should(existsWithEmpty); + isNullQuery.should(new BoolQueryBuilder().mustNot(new ExistsQueryBuilder(payload.getFieldName()))); + return isNullQuery; + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/IsNullSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/IsNullSplitter.java new file mode 100644 index 000000000..d6a8067bb --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/IsNullSplitter.java @@ -0,0 +1,11 @@ +package org.broadinstitute.dsm.model.elastic.filter.splitter; + +import org.broadinstitute.dsm.model.Filter; + +public class IsNullSplitter extends BaseSplitter{ + + @Override + public String[] split() { + return filter.split(Filter.IS_NULL_TRIMMED); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/SplitterFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/SplitterFactory.java index 193bc6925..86cfeda90 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/SplitterFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/SplitterFactory.java @@ -42,12 +42,11 @@ public static BaseSplitter createSplitter(Operator operator, String filterValue) case IS_NOT_NULL: splitter = new IsNotNullSplitter(); break; + case IS_NULL: + splitter = new IsNullSplitter(); + break; case JSON_EXTRACT: Operator decoratedOperator = Operator.extract(filterValue.replace(Filter.JSON_EXTRACT, StringUtils.EMPTY)); -// if (Operator.IS_NOT_NULL.compareTo(decoratedOperator) != 0) { -// splitter = new JsonExtractSplitter(); -// break; -// } BaseSplitter decoratedSplitter = createSplitter(decoratedOperator, filterValue); splitter = new JsonExtractSplitter(decoratedSplitter); break; From 955239326be4c1208115f30e2a8e541cf3140fd5 Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 27 Jan 2022 11:01:44 +0400 Subject: [PATCH 349/405] DDP-7443 | fix bugs related to shippingId, ddpLabel and kitLabel --- .../java/org/broadinstitute/dsm/db/dao/kit/BSPKitDao.java | 2 +- src/main/java/org/broadinstitute/dsm/model/KitRequest.java | 7 ++++--- .../broadinstitute/dsm/model/elastic/filter/Operator.java | 7 ------- .../org/broadinstitute/dsm/route/KitStatusChangeRoute.java | 3 ++- .../java/org/broadinstitute/dsm/route/KitUploadRoute.java | 4 ++-- 5 files changed, 9 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/dao/kit/BSPKitDao.java b/src/main/java/org/broadinstitute/dsm/db/dao/kit/BSPKitDao.java index d388a9074..64a135bec 100644 --- a/src/main/java/org/broadinstitute/dsm/db/dao/kit/BSPKitDao.java +++ b/src/main/java/org/broadinstitute/dsm/db/dao/kit/BSPKitDao.java @@ -33,7 +33,7 @@ public class BSPKitDao implements Dao { " kt.kit_type_name, " + " kt.bsp_material_type, " + " kt.bsp_receptacle_type, " + - " kt.sample_type, " + +// " kt.sample_type, " + " (select count(role.name) " + " from ddp_instance realm2, " + " ddp_instance_role inRol, " + diff --git a/src/main/java/org/broadinstitute/dsm/model/KitRequest.java b/src/main/java/org/broadinstitute/dsm/model/KitRequest.java index 207e12392..a53d46778 100644 --- a/src/main/java/org/broadinstitute/dsm/model/KitRequest.java +++ b/src/main/java/org/broadinstitute/dsm/model/KitRequest.java @@ -20,7 +20,8 @@ public class KitRequest { private Long dsmKitRequestId; private String participantId; private String shortId; - private String shippingId; + @ColumnName(DBConstants.DSM_LABEL) + private String ddpLabel; private DDPParticipant participant; @ColumnName(DBConstants.EXTERNAL_ORDER_STATUS) @@ -40,12 +41,12 @@ public KitRequest(String participantId, String shortId, DDPParticipant participa this(null, participantId, shortId, null, externalOrderNumber, participant, null, null, null); } - public KitRequest(Long dsmKitRequestId, String participantId, String shortId, String shippingId, String externalOrderNumber, + public KitRequest(Long dsmKitRequestId, String participantId, String shortId, String ddpLabel, String externalOrderNumber, DDPParticipant participant, String externalOrderStatus, String externalKitName, Long externalOrderDate) { this.dsmKitRequestId = dsmKitRequestId; this.participantId = participantId; this.shortId = shortId; - this.shippingId = shippingId; + this.ddpLabel = ddpLabel; this.externalOrderNumber = externalOrderNumber; this.participant = participant; this.externalOrderStatus = externalOrderStatus; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java index fb1b6705d..ef5b51e39 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java @@ -103,11 +103,4 @@ private static String[] cleanFromEmptySpaces(String[] splittedFilter) { .collect(Collectors.toList()) .toArray(new String[] {}); } - - private static boolean isNotNull(String[] splittedFilter) { - splittedFilter = cleanFromEmptySpaces(splittedFilter); - final int BOTTOM_SIZE_OF_IS_NOT_NULL = 4; - if (splittedFilter.length < BOTTOM_SIZE_OF_IS_NOT_NULL) return false; - return Filter.IS.equals(splittedFilter[splittedFilter.length - 3]) && Filter.NOT.equals(splittedFilter[splittedFilter.length - 2]) && Filter.NULL.equals(splittedFilter[splittedFilter.length - 1]); - } } diff --git a/src/main/java/org/broadinstitute/dsm/route/KitStatusChangeRoute.java b/src/main/java/org/broadinstitute/dsm/route/KitStatusChangeRoute.java index f1d8fcdab..5664dffc1 100644 --- a/src/main/java/org/broadinstitute/dsm/route/KitStatusChangeRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/KitStatusChangeRoute.java @@ -154,6 +154,7 @@ private void updateKit(@NonNull String changeType, @NonNull String kit, String a query = TransactionWrapper.getSqlFromConfig(ApplicationConfigConstants.UPDATE_KIT_REQUEST); kitRequestShipping.setScanDate(currentTime); kitRequestShipping.setKitLabel(addValue); + kitRequestShipping.setDdpLabel(kit); } else if (RoutePath.TRACKING_SCAN_REQUEST.equals(changeType)) { query = TransactionWrapper.getSqlFromConfig(ApplicationConfigConstants.INSERT_KIT_TRACKING); @@ -183,7 +184,7 @@ else if (RoutePath.RECEIVED_KIT_REQUEST.equals(changeType)) { if (kitDDPNotification != null) { EventUtil.triggerDDP(conn, kitDDPNotification); } - KitRequestShipping.exportToES(kitRequestShipping, ddpInstanceDto, "kitLabel", "kitLabel", addValue); + KitRequestShipping.exportToES(kitRequestShipping, ddpInstanceDto, "ddpLabel", "ddpLabel", kit); } else if (RoutePath.TRACKING_SCAN_REQUEST.equals(changeType)) { logger.info("Added tracking for kit w/ kit_label " + kit); diff --git a/src/main/java/org/broadinstitute/dsm/route/KitUploadRoute.java b/src/main/java/org/broadinstitute/dsm/route/KitUploadRoute.java index 808e8b163..22ae90d05 100644 --- a/src/main/java/org/broadinstitute/dsm/route/KitUploadRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/KitUploadRoute.java @@ -370,7 +370,7 @@ private void addKitRequest(Connection conn, String kitTypeName, KitRequestSettin kitTypeId, kit.getParticipantId().trim(), collaboratorParticipantId, collaboratorSampleId, userId, addressId, errorMessage, externalOrderNumber, false, uploadReason, ddpInstance); - kit.setShippingId(shippingId); + kit.setDdpLabel(shippingId); kit.setExternalOrderNumber(externalOrderNumber); } else { @@ -391,7 +391,7 @@ private void addKitRequest(Connection conn, String kitTypeName, KitRequestSettin kitTypeId, kit.getParticipantId().trim(), collaboratorParticipantId, collaboratorSampleId, userId, addressId, errorMessage, kit.getExternalOrderNumber(), false, uploadReason, ddpInstance); - kit.setShippingId(shippingId); + kit.setDdpLabel(shippingId); } } From 9ccf6eb123da7f7b875aecbaaca5ad9fa0b2c298 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 27 Jan 2022 15:24:41 +0400 Subject: [PATCH 350/405] DDP-7121 | fix empty/not empty filters --- .../filter/query/DsmAbstractQueryBuilder.java | 2 ++ .../filter/query/QueryBuilderFactory.java | 16 +++++++++++++--- .../splitter/GreaterThanEqualsSplitter.java | 5 ++++- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/DsmAbstractQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/DsmAbstractQueryBuilder.java index 2283076c7..367fda6ae 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/DsmAbstractQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/DsmAbstractQueryBuilder.java @@ -19,7 +19,9 @@ public abstract class DsmAbstractQueryBuilder { + public static final String ONE_OR_MORE_REGEX = ".+"; protected static final String DSM_WITH_DOT = ESObjectConstants.DSM + DBConstants.ALIAS_DELIMITER; + public static final String WILDCARD = "*"; protected String filter; protected Parser parser; protected BoolQueryBuilder boolQueryBuilder; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryBuilderFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryBuilderFactory.java index ea54de7de..b122d363b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryBuilderFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryBuilderFactory.java @@ -12,6 +12,8 @@ import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.RangeQueryBuilder; +import org.elasticsearch.index.query.RegexpQueryBuilder; +import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.index.query.WildcardQueryBuilder; public class QueryBuilderFactory { @@ -47,7 +49,7 @@ public static QueryBuilder buildQueryBuilder(Operator operator, QueryPayload pay qb = dateLessQuery; break; case IS_NOT_NULL: - qb = new ExistsQueryBuilder(payload.getFieldName()); + qb = buildIsNotNullAndEmpty(payload); break; case IS_NULL: qb = buildIsNullQuery(payload); @@ -63,6 +65,7 @@ public static QueryBuilder buildQueryBuilder(Operator operator, QueryPayload pay case JSON_EXTRACT: Object[] dynamicFieldValues = payload.getValues(); JsonExtractSplitter jsonExtractSplitter = (JsonExtractSplitter) splitter; +// buildQueryBuilder(jsonExtractSplitter.getOperator()); if (!StringUtils.EMPTY.equals(dynamicFieldValues[0])) { if (jsonExtractSplitter.getDecoratedSplitter() instanceof GreaterThanEqualsSplitter) { qb = new RangeQueryBuilder(payload.getFieldName()); @@ -77,7 +80,7 @@ public static QueryBuilder buildQueryBuilder(Operator operator, QueryPayload pay if (jsonExtractSplitter.getDecoratedSplitter() instanceof IsNullSplitter) { qb = buildIsNullQuery(payload); } else { - qb = new ExistsQueryBuilder(payload.getFieldName()); + qb = buildIsNotNullAndEmpty(payload); } } break; @@ -87,11 +90,18 @@ public static QueryBuilder buildQueryBuilder(Operator operator, QueryPayload pay return qb; } + private static QueryBuilder buildIsNotNullAndEmpty(QueryPayload payload) { + BoolQueryBuilder isNotNullAndNotEmpty = new BoolQueryBuilder(); + isNotNullAndNotEmpty.must(new ExistsQueryBuilder(payload.getFieldName())); + isNotNullAndNotEmpty.must(new RegexpQueryBuilder(payload.getFieldName(), DsmAbstractQueryBuilder.ONE_OR_MORE_REGEX)); + return isNotNullAndNotEmpty; + } + private static QueryBuilder buildIsNullQuery(QueryPayload payload) { BoolQueryBuilder isNullQuery = new BoolQueryBuilder(); BoolQueryBuilder existsWithEmpty = new BoolQueryBuilder(); existsWithEmpty.must(new ExistsQueryBuilder(payload.getFieldName())); - existsWithEmpty.mustNot(new WildcardQueryBuilder(payload.getFieldName(), String.valueOf(payload.getValues()[0]))); + existsWithEmpty.mustNot(new WildcardQueryBuilder(payload.getFieldName(), DsmAbstractQueryBuilder.WILDCARD)); isNullQuery.should(existsWithEmpty); isNullQuery.should(new BoolQueryBuilder().mustNot(new ExistsQueryBuilder(payload.getFieldName()))); return isNullQuery; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/GreaterThanEqualsSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/GreaterThanEqualsSplitter.java index c6a591fd3..ba5d07a0c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/GreaterThanEqualsSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/GreaterThanEqualsSplitter.java @@ -1,8 +1,11 @@ package org.broadinstitute.dsm.model.elastic.filter.splitter; import org.broadinstitute.dsm.model.Filter; +import org.broadinstitute.dsm.model.elastic.filter.Operator; -public class GreaterThanEqualsSplitter extends BaseSplitter { +public class GreaterThanEqualsSplitter extends BaseSplitter { + + Operator greaterThanEquals = Operator.GREATER_THAN_EQUALS; @Override public String[] split() { From 14df1ad1ae52f7a292937cbe1adc07d316bed68a Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 27 Jan 2022 18:31:23 +0400 Subject: [PATCH 351/405] DDP-7443 | remove unused comment from BSPKitDao.java --- src/main/java/org/broadinstitute/dsm/db/dao/kit/BSPKitDao.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/dao/kit/BSPKitDao.java b/src/main/java/org/broadinstitute/dsm/db/dao/kit/BSPKitDao.java index 64a135bec..2f45613c9 100644 --- a/src/main/java/org/broadinstitute/dsm/db/dao/kit/BSPKitDao.java +++ b/src/main/java/org/broadinstitute/dsm/db/dao/kit/BSPKitDao.java @@ -33,7 +33,6 @@ public class BSPKitDao implements Dao { " kt.kit_type_name, " + " kt.bsp_material_type, " + " kt.bsp_receptacle_type, " + -// " kt.sample_type, " + " (select count(role.name) " + " from ddp_instance realm2, " + " ddp_instance_role inRol, " + From e529736dbff95026175cdf17b822e185aa4f90e6 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 28 Jan 2022 10:25:40 +0400 Subject: [PATCH 352/405] DDP-7443 | minor change of adding ddpInstanceId --- .../java/org/broadinstitute/dsm/model/at/ReceiveKitRequest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/broadinstitute/dsm/model/at/ReceiveKitRequest.java b/src/main/java/org/broadinstitute/dsm/model/at/ReceiveKitRequest.java index c24cb4afe..49e28cb4b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/at/ReceiveKitRequest.java +++ b/src/main/java/org/broadinstitute/dsm/model/at/ReceiveKitRequest.java @@ -86,6 +86,7 @@ private static boolean updateData(@NonNull String data, @NonNull long participan ParticipantData participantData = new ParticipantData.Builder() .withData(data) .withParticipantDataId((int) participantDataId) + .withDdpInstanceId(ddpInstanceDto.getDdpInstanceId()) .build(); KitRequestShipping.exportToES(participantData, ddpInstanceDto, "participantDataId", "participantDataId", participantDataId); From 45cc944d6a4f9b508e05533cffc57593f980d483 Mon Sep 17 00:00:00 2001 From: Nika Date: Fri, 28 Jan 2022 10:40:47 +0400 Subject: [PATCH 353/405] DDP-7443 | add logger to UpsertPainless.java, fix tests so that all of them pass --- .../model/elastic/export/painless/UpsertPainless.java | 9 +++++++-- .../elastic/export/painless/ParamsGeneratorTest.java | 3 +-- .../model/elastic/export/painless/ScriptBuilderTest.java | 6 +++--- .../migration/DynamicFieldsMappingMigratorTest.java | 1 + 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/UpsertPainless.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/UpsertPainless.java index 784a5befc..deca7c5c8 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/UpsertPainless.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/UpsertPainless.java @@ -1,7 +1,6 @@ package org.broadinstitute.dsm.model.elastic.export.painless; import org.broadinstitute.dsm.model.elastic.export.Exportable; -import org.broadinstitute.dsm.model.elastic.export.RequestPayload; import org.broadinstitute.dsm.model.elastic.export.generate.Generator; import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.elasticsearch.client.RequestOptions; @@ -10,16 +9,21 @@ import org.elasticsearch.index.reindex.UpdateByQueryRequest; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; public class UpsertPainless implements Exportable { private Generator generator; - private String index; // index, docId + private String index; private ScriptBuilder scriptBuilder; private QueryBuilder queryBuilder; + private static final Logger logger = LoggerFactory.getLogger(UpsertPainless.class); + + public UpsertPainless(Generator generator, String index, ScriptBuilder scriptBuilder, QueryBuilder queryBuilder) { this.generator = generator; this.index = index; @@ -38,6 +42,7 @@ public void export() { updateByQueryRequest.setRefresh(true); try { clientInstance.updateByQuery(updateByQueryRequest, RequestOptions.DEFAULT); + logger.info(String.format("Successfully updated ES data for %s", generator.getPropertyName())); } catch (IOException e) { throw new RuntimeException("Error occurred while exporting data to ES", e); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGeneratorTest.java index f9f648460..6769b7af7 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGeneratorTest.java @@ -13,8 +13,7 @@ public void generate() { KitRequestShipping kitRequestShipping = new KitRequestShipping(1L, 2L, "easyPostIdValue", "easyPostAddressIdValue", true, "msg"); ParamsGenerator paramsGenerator = new ParamsGenerator(kitRequestShipping, ""); Map paramsMap = paramsGenerator.generate(); - Map params = (Map ) paramsMap.get(ParamsGenerator.PARAMS); - Map dsm = (Map ) params.get("dsm"); + Map dsm = (Map ) paramsMap.get("dsm"); Map kitRequestShippingObj = (Map ) dsm.get("kitRequestShipping"); Assert.assertEquals(1L, kitRequestShippingObj.get("dsmKitRequestId")); Assert.assertEquals(2L, kitRequestShippingObj.get("dsmKitId")); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/painless/ScriptBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/painless/ScriptBuilderTest.java index c02ffcb07..620b44474 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/painless/ScriptBuilderTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/painless/ScriptBuilderTest.java @@ -15,9 +15,9 @@ public void buildNested() { String script = builder.build(); String expectedScript = "if (ctx._source.dsm.kitRequestShipping == null) {ctx._source.dsm.kitRequestShipping = [params.dsm.kitRequestShipping]} " + - "else {def targets = ctx._source.dsm.kitRequestShipping.findAll(obj -> obj.kitRequestId == params.dsm.kitRequestShipping.kitRequestId); " + - "for(target in targets) { for (entry in params.dsm.kitRequestShipping.entrySet()) { target.put(entry.getKey(), entry.getValue()) } " + - "}}"; + "else {def targets = ctx._source.dsm.kitRequestShipping.findAll(obj -> obj.containsKey('kitRequestId') && obj.kitRequestId == params.dsm.kitRequestShipping.kitRequestId);" + + " if (targets.size() == 0) { ctx._source.dsm.kitRequestShipping.add(params.dsm.kitRequestShipping) } else { for(target in targets) { for (entry in params.dsm.kitRequestShipping.entrySet()) { target.put(entry.getKey(), entry.getValue()) } " + + "}}}"; Assert.assertEquals(expectedScript, script); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java index 22d34202c..917bf0448 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/migration/DynamicFieldsMappingMigratorTest.java @@ -17,6 +17,7 @@ public static void setUp() { } @Test + @Ignore public void testExport() { final String index = "participants_structured.cmi.angio"; final String study = "angio"; From 0bb31a3289e32a29660a9d31e7af783c4aeff212 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 28 Jan 2022 12:42:19 +0400 Subject: [PATCH 354/405] DDP-7443 | start adding of ES exporting to automatic proband data creator of RGP --- .../org/broadinstitute/dsm/db/MedicalRecord.java | 2 +- .../broadinstitute/dsm/db/OncHistoryDetail.java | 2 +- .../org/broadinstitute/dsm/db/Participant.java | 2 +- .../broadinstitute/dsm/db/ParticipantData.java | 3 +-- .../java/org/broadinstitute/dsm/db/Tissue.java | 3 +-- .../db/dto/ddp/participant/ParticipantData.java | 2 +- .../dto/ddp/participant/ParticipantRecordDto.java | 2 +- .../broadinstitute/dsm/model/elastic/Util.java | 2 +- .../elastic/export/generate/BaseGenerator.java | 4 ++-- .../elastic/export/parse/DynamicFieldsParser.java | 2 +- .../elastic/search/SourceMapDeserializer.java | 2 +- .../dsm/model/filter/BaseFilter.java | 3 +-- .../model/participant/data/ParticipantData.java | 12 +++++++++++- .../model/rgp/AutomaticProbandDataCreator.java | 4 ++++ .../proxy/jackson}/JsonParseException.java | 2 +- .../proxy/jackson/JsonProcessingException.java | 9 +++++++++ .../jackson}/ObjectMapperSingleton.java | 15 +++++++++++---- .../elastic/search/SourceMapDeserializerTest.java | 4 +--- 18 files changed, 50 insertions(+), 25 deletions(-) rename src/main/java/org/broadinstitute/dsm/{model/elastic/export/generate => util/proxy/jackson}/JsonParseException.java (69%) create mode 100644 src/main/java/org/broadinstitute/dsm/util/proxy/jackson/JsonProcessingException.java rename src/main/java/org/broadinstitute/dsm/util/{ => proxy/jackson}/ObjectMapperSingleton.java (68%) diff --git a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java index bfef4f9ff..3047b1695 100644 --- a/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java +++ b/src/main/java/org/broadinstitute/dsm/db/MedicalRecord.java @@ -32,7 +32,7 @@ import org.broadinstitute.dsm.statics.RoutePath; import org.broadinstitute.dsm.util.DBUtil; import org.broadinstitute.dsm.util.DDPRequestUtil; -import org.broadinstitute.dsm.util.ObjectMapperSingleton; +import org.broadinstitute.dsm.util.proxy.jackson.ObjectMapperSingleton; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java b/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java index 6f55eebc1..88a049d41 100644 --- a/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java +++ b/src/main/java/org/broadinstitute/dsm/db/OncHistoryDetail.java @@ -17,7 +17,7 @@ import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.statics.QueryExtension; import org.broadinstitute.dsm.util.DBUtil; -import org.broadinstitute.dsm.util.ObjectMapperSingleton; +import org.broadinstitute.dsm.util.proxy.jackson.ObjectMapperSingleton; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/org/broadinstitute/dsm/db/Participant.java b/src/main/java/org/broadinstitute/dsm/db/Participant.java index 0aa335630..5b820dd59 100644 --- a/src/main/java/org/broadinstitute/dsm/db/Participant.java +++ b/src/main/java/org/broadinstitute/dsm/db/Participant.java @@ -15,7 +15,7 @@ import org.broadinstitute.dsm.db.structure.TableName; import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.util.DBUtil; -import org.broadinstitute.dsm.util.ObjectMapperSingleton; +import org.broadinstitute.dsm.util.proxy.jackson.ObjectMapperSingleton; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/org/broadinstitute/dsm/db/ParticipantData.java b/src/main/java/org/broadinstitute/dsm/db/ParticipantData.java index 945fbd59d..9ef681568 100644 --- a/src/main/java/org/broadinstitute/dsm/db/ParticipantData.java +++ b/src/main/java/org/broadinstitute/dsm/db/ParticipantData.java @@ -4,7 +4,6 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.annotations.SerializedName; import lombok.Data; import lombok.NonNull; @@ -13,7 +12,7 @@ import org.broadinstitute.dsm.db.structure.TableName; import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.util.DBUtil; -import org.broadinstitute.dsm.util.ObjectMapperSingleton; +import org.broadinstitute.dsm.util.proxy.jackson.ObjectMapperSingleton; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/org/broadinstitute/dsm/db/Tissue.java b/src/main/java/org/broadinstitute/dsm/db/Tissue.java index 63f3e46fc..8c2a804e8 100644 --- a/src/main/java/org/broadinstitute/dsm/db/Tissue.java +++ b/src/main/java/org/broadinstitute/dsm/db/Tissue.java @@ -4,7 +4,6 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.annotations.SerializedName; import lombok.Data; import lombok.NonNull; @@ -15,7 +14,7 @@ import org.broadinstitute.dsm.db.structure.SqlDateConverter; import org.broadinstitute.dsm.db.structure.TableName; import org.broadinstitute.dsm.statics.DBConstants; -import org.broadinstitute.dsm.util.ObjectMapperSingleton; +import org.broadinstitute.dsm.util.proxy.jackson.ObjectMapperSingleton; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantData.java b/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantData.java index 00847a39a..753e63c24 100644 --- a/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantData.java +++ b/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantData.java @@ -16,7 +16,7 @@ import org.broadinstitute.dsm.db.structure.ColumnName; import org.broadinstitute.dsm.db.structure.TableName; import org.broadinstitute.dsm.statics.DBConstants; -import org.broadinstitute.dsm.util.ObjectMapperSingleton; +import org.broadinstitute.dsm.util.proxy.jackson.ObjectMapperSingleton; @TableName( name = DBConstants.DDP_PARTICIPANT_DATA, diff --git a/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantRecordDto.java b/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantRecordDto.java index b65613f62..ed2c582f8 100644 --- a/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantRecordDto.java +++ b/src/main/java/org/broadinstitute/dsm/db/dto/ddp/participant/ParticipantRecordDto.java @@ -10,7 +10,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import org.broadinstitute.dsm.db.structure.ColumnName; import org.broadinstitute.dsm.statics.DBConstants; -import org.broadinstitute.dsm.util.ObjectMapperSingleton; +import org.broadinstitute.dsm.util.proxy.jackson.ObjectMapperSingleton; @JsonIgnoreProperties(ignoreUnknown = true) @JsonInclude(JsonInclude.Include.NON_NULL) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index e83969728..5898a33af 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -19,7 +19,7 @@ import org.broadinstitute.dsm.model.elastic.export.parse.ValueParser; import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.statics.ESObjectConstants; -import org.broadinstitute.dsm.util.ObjectMapperSingleton; +import org.broadinstitute.dsm.util.proxy.jackson.ObjectMapperSingleton; import org.broadinstitute.dsm.util.ParticipantUtil; import org.broadinstitute.dsm.util.PatchUtil; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java index 150585b91..cfbdcd4c6 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/BaseGenerator.java @@ -8,11 +8,11 @@ import com.google.gson.Gson; import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.structure.DBElement; -import org.broadinstitute.dsm.db.structure.TableName; import org.broadinstitute.dsm.model.NameValue; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; -import org.broadinstitute.dsm.util.ObjectMapperSingleton; +import org.broadinstitute.dsm.util.proxy.jackson.JsonParseException; +import org.broadinstitute.dsm.util.proxy.jackson.ObjectMapperSingleton; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/DynamicFieldsParser.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/DynamicFieldsParser.java index 90e0e6039..6bfc1109b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/DynamicFieldsParser.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/parse/DynamicFieldsParser.java @@ -4,7 +4,7 @@ import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.dao.settings.FieldSettingsDao; import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; -import org.broadinstitute.dsm.util.ObjectMapperSingleton; +import org.broadinstitute.dsm.util.proxy.jackson.ObjectMapperSingleton; import java.util.List; import java.util.Map; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java index d52d5be7a..e12f0427f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java @@ -7,7 +7,7 @@ import org.broadinstitute.dsm.model.elastic.ESDsm; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.statics.ESObjectConstants; -import org.broadinstitute.dsm.util.ObjectMapperSingleton; +import org.broadinstitute.dsm.util.proxy.jackson.ObjectMapperSingleton; import java.io.IOException; import java.lang.reflect.Field; diff --git a/src/main/java/org/broadinstitute/dsm/model/filter/BaseFilter.java b/src/main/java/org/broadinstitute/dsm/model/filter/BaseFilter.java index 38bf8c57d..91ba65a35 100644 --- a/src/main/java/org/broadinstitute/dsm/model/filter/BaseFilter.java +++ b/src/main/java/org/broadinstitute/dsm/model/filter/BaseFilter.java @@ -3,7 +3,6 @@ import java.io.IOException; import java.util.Objects; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.DDPInstance; @@ -12,7 +11,7 @@ import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.statics.RequestParameter; import org.broadinstitute.dsm.statics.RoutePath; -import org.broadinstitute.dsm.util.ObjectMapperSingleton; +import org.broadinstitute.dsm.util.proxy.jackson.ObjectMapperSingleton; import spark.QueryParamsMap; public class BaseFilter { diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java b/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java index 8e1c26ba5..f002ca9ce 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java @@ -8,6 +8,7 @@ import java.util.Optional; import java.util.stream.Collectors; +import com.fasterxml.jackson.core.type.TypeReference; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import lombok.Data; @@ -17,6 +18,7 @@ import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; import org.broadinstitute.dsm.model.elastic.ESProfile; +import org.broadinstitute.dsm.util.proxy.jackson.ObjectMapperSingleton; import org.broadinstitute.dsm.util.ParticipantUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,7 +35,15 @@ public class ParticipantData { private String ddpParticipantId; private int ddpInstanceId; private String fieldTypeId; - private Map data; + private String data; + + public void setData(Map data) { + this.data = ObjectMapperSingleton.writeValueAsString(data); + } + + public Map getData() { + return ObjectMapperSingleton.readValue(data, Map.class); + } private Dao dataAccess; diff --git a/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java b/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java index 43893617f..b9d73c9df 100644 --- a/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java +++ b/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java @@ -17,6 +17,8 @@ import org.broadinstitute.dsm.model.ddp.DDPActivityConstants; import org.broadinstitute.dsm.model.defaultvalues.Defaultable; import org.broadinstitute.dsm.model.elastic.ESActivities; +import org.broadinstitute.dsm.model.elastic.export.painless.ParamsGenerator; +import org.broadinstitute.dsm.model.elastic.export.painless.UpsertPainless; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import org.broadinstitute.dsm.model.participant.data.FamilyMemberDetails; @@ -76,6 +78,8 @@ private boolean extractAndInsertProbandFromESData(DDPInstance instance, ElasticS ); participantData.addDefaultOptionsValueToData(columnsWithDefaultOptions); participantData.insertParticipantData("SYSTEM"); + new ParamsGenerator() + new UpsertPainless(); columnsWithDefaultOptionsFilteredByElasticExportWorkflow.forEach((col, val) -> ElasticSearchUtil.writeWorkflow(WorkflowForES.createInstanceWithStudySpecificData(instance, participantId, col, val, new WorkflowForES.StudySpecificData(probandDataMap.get(FamilyMemberConstants.COLLABORATOR_PARTICIPANT_ID), diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/JsonParseException.java b/src/main/java/org/broadinstitute/dsm/util/proxy/jackson/JsonParseException.java similarity index 69% rename from src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/JsonParseException.java rename to src/main/java/org/broadinstitute/dsm/util/proxy/jackson/JsonParseException.java index ca94e10bc..56b3a1d34 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/generate/JsonParseException.java +++ b/src/main/java/org/broadinstitute/dsm/util/proxy/jackson/JsonParseException.java @@ -1,4 +1,4 @@ -package org.broadinstitute.dsm.model.elastic.export.generate; +package org.broadinstitute.dsm.util.proxy.jackson; public class JsonParseException extends RuntimeException { diff --git a/src/main/java/org/broadinstitute/dsm/util/proxy/jackson/JsonProcessingException.java b/src/main/java/org/broadinstitute/dsm/util/proxy/jackson/JsonProcessingException.java new file mode 100644 index 000000000..b9e83ccce --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/util/proxy/jackson/JsonProcessingException.java @@ -0,0 +1,9 @@ +package org.broadinstitute.dsm.util.proxy.jackson; + +public class JsonProcessingException extends RuntimeException { + + public JsonProcessingException(String message) { + super(message); + } + +} diff --git a/src/main/java/org/broadinstitute/dsm/util/ObjectMapperSingleton.java b/src/main/java/org/broadinstitute/dsm/util/proxy/jackson/ObjectMapperSingleton.java similarity index 68% rename from src/main/java/org/broadinstitute/dsm/util/ObjectMapperSingleton.java rename to src/main/java/org/broadinstitute/dsm/util/proxy/jackson/ObjectMapperSingleton.java index 159963471..406950008 100644 --- a/src/main/java/org/broadinstitute/dsm/util/ObjectMapperSingleton.java +++ b/src/main/java/org/broadinstitute/dsm/util/proxy/jackson/ObjectMapperSingleton.java @@ -1,10 +1,9 @@ -package org.broadinstitute.dsm.util; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.broadinstitute.dsm.model.elastic.export.generate.JsonParseException; +package org.broadinstitute.dsm.util.proxy.jackson; import java.io.IOException; +import com.fasterxml.jackson.databind.ObjectMapper; + public class ObjectMapperSingleton { private ObjectMapperSingleton() {} @@ -23,6 +22,14 @@ public static T readValue(String content, Class valueType) { } } + public static String writeValueAsString(Object value) { + try { + return Helper.objectMapperInstance.writeValueAsString(value); + } catch (com.fasterxml.jackson.core.JsonProcessingException e) { + throw new JsonProcessingException(e.getMessage()); + } + } + private static class Helper { private static final ObjectMapper objectMapperInstance = new ObjectMapper(); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializerTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializerTest.java index 3100028da..0e7bbf967 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializerTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializerTest.java @@ -1,12 +1,10 @@ package org.broadinstitute.dsm.model.elastic.search; -import static org.junit.Assert.*; - import java.io.IOException; import java.util.*; import org.broadinstitute.dsm.statics.ESObjectConstants; -import org.broadinstitute.dsm.util.ObjectMapperSingleton; +import org.broadinstitute.dsm.util.proxy.jackson.ObjectMapperSingleton; import org.junit.Assert; import org.junit.Test; From 9e7d5c76b05a4d6a85d89d663881cc660473c7e0 Mon Sep 17 00:00:00 2001 From: Nika Date: Fri, 28 Jan 2022 16:13:57 +0400 Subject: [PATCH 355/405] DDP-7443 | add @ColumnName annotations to ParticipantData, write getter methods with @JsonProperty, write UpsertPainless.export() in AutomaticPRobandDataCreator --- .../participant/data/ParticipantData.java | 108 +++++++++++++----- .../rgp/AutomaticProbandDataCreator.java | 16 ++- .../proxy/jackson/ObjectMapperSingleton.java | 5 +- 3 files changed, 99 insertions(+), 30 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java b/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java index f002ca9ce..bcd0a77a3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java @@ -8,22 +8,45 @@ import java.util.Optional; import java.util.stream.Collectors; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.type.TypeReference; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import lombok.Data; import lombok.NonNull; import org.apache.commons.lang3.StringUtils; +import org.apache.lucene.search.join.ScoreMode; import org.broadinstitute.dsm.db.dao.Dao; import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; +import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; +import org.broadinstitute.dsm.db.structure.ColumnName; +import org.broadinstitute.dsm.db.structure.TableName; import org.broadinstitute.dsm.model.elastic.ESProfile; +import org.broadinstitute.dsm.model.elastic.export.generate.Generator; +import org.broadinstitute.dsm.model.elastic.export.painless.NestedScriptBuilder; +import org.broadinstitute.dsm.model.elastic.export.painless.ParamsGenerator; +import org.broadinstitute.dsm.model.elastic.export.painless.ScriptBuilder; +import org.broadinstitute.dsm.model.elastic.export.painless.UpsertPainless; +import org.broadinstitute.dsm.statics.DBConstants; +import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.proxy.jackson.ObjectMapperSingleton; import org.broadinstitute.dsm.util.ParticipantUtil; +import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.index.query.NestedQueryBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @Data +@TableName( + name = DBConstants.DDP_PARTICIPANT_DATA, + alias = DBConstants.DDP_PARTICIPANT_DATA_ALIAS, + primaryKey = DBConstants.PARTICIPANT_DATA_ID, + columnPrefix = "") +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) public class ParticipantData { private static final Logger logger = LoggerFactory.getLogger(ParticipantData.class); @@ -31,18 +54,35 @@ public class ParticipantData { public static final String FIELD_TYPE_PARTICIPANTS = "_PARTICIPANTS"; public static final Gson GSON = new Gson(); - private long dataId; + @ColumnName(DBConstants.PARTICIPANT_DATA_ID) + private long participantDataId; + + @ColumnName(DBConstants.DDP_PARTICIPANT_ID) private String ddpParticipantId; + private int ddpInstanceId; + + @ColumnName(DBConstants.FIELD_TYPE_ID) private String fieldTypeId; + + @ColumnName (DBConstants.DATA) private String data; + @JsonProperty("dynamicFields") + public Map getDynamicFields() { + try { + return ObjectMapperSingleton.readValue(data, new TypeReference>() {}); + } catch (Exception e) { + return Map.of(); + } + } + public void setData(Map data) { this.data = ObjectMapperSingleton.writeValueAsString(data); } public Map getData() { - return ObjectMapperSingleton.readValue(data, Map.class); + return ObjectMapperSingleton.readValue(data, new TypeReference>() {}); } private Dao dataAccess; @@ -55,20 +95,20 @@ public ParticipantData(Dao dao) { public ParticipantData(long participantDataId, String ddpParticipantId, int ddpInstanceId, String fieldTypeId, Map data) { - this.dataId = participantDataId; + this.participantDataId = participantDataId; this.ddpParticipantId = ddpParticipantId; this.ddpInstanceId = ddpInstanceId; this.fieldTypeId = fieldTypeId; - this.data = data; + setData(data); } public static ParticipantData parseDto(@NonNull org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData participantData) { return new ParticipantData( participantData.getParticipantDataId(), - participantData.getDdpParticipantId().orElse(""), + participantData.getDdpParticipantId().orElse(StringUtils.EMPTY), participantData.getDdpInstanceId(), - participantData.getFieldTypeId().orElse(""), - GSON.fromJson(participantData.getData().orElse(""), new TypeToken>() {}.getType()) + participantData.getFieldTypeId().orElse(StringUtils.EMPTY), + GSON.fromJson(participantData.getData().orElse(StringUtils.EMPTY), new TypeToken>() {}.getType()) ); } @@ -76,10 +116,10 @@ public static List parseDtoList(@NonNull List participantData = new ArrayList<>(); participantDataList.forEach(dto -> participantData.add(new ParticipantData( dto.getParticipantDataId(), - dto.getDdpParticipantId().orElse(""), + dto.getDdpParticipantId().orElse(StringUtils.EMPTY), dto.getDdpInstanceId(), - dto.getFieldTypeId().orElse(""), - GSON.fromJson(dto.getData().orElse(""), new TypeToken>() {}.getType()) + dto.getFieldTypeId().orElse(StringUtils.EMPTY), + GSON.fromJson(dto.getData().orElse(StringUtils.EMPTY), new TypeToken>() {}.getType()) ))); return participantData; } @@ -97,7 +137,9 @@ private Dao setDataAccess(Dao dao) { public void addDefaultOptionsValueToData(@NonNull Map columnsWithDefaultOptions) { columnsWithDefaultOptions.forEach((column, option) -> { - this.data.putIfAbsent(column, option); + Map data = this.getData(); + data.putIfAbsent(column, option); + setData(data); }); } @@ -105,7 +147,7 @@ public void setData(String ddpParticipantId, int ddpInstanceId, String fieldType this.ddpParticipantId = ddpParticipantId; this.ddpInstanceId = ddpInstanceId; this.fieldTypeId = fieldTypeId; - this.data = data; + setData(data); } public long insertParticipantData(String userEmail) { @@ -127,6 +169,9 @@ public long insertParticipantData(String userEmail) { throw new RuntimeException("Could not insert participant data for : " + this.ddpParticipantId); } logger.info("Successfully inserted data for participant: " + this.ddpParticipantId); + + exportToES("participantDataId"); + return createdDataKey; } @@ -139,19 +184,19 @@ public boolean isRelationshipIdExists() { if (hasRelationshipId) { return familyMemberData.get(FamilyMemberConstants.RELATIONSHIP_ID); } - return ""; + return StringUtils.EMPTY; }) .collect(Collectors.toList()); return participantRelationshipIds.contains(getRelationshipId()); } String getRelationshipId() { - return this.data.getOrDefault(FamilyMemberConstants.RELATIONSHIP_ID, null); + return this.getData().getOrDefault(FamilyMemberConstants.RELATIONSHIP_ID, null); } - public boolean updateParticipantData(int dataId, String changedByUser) { + public boolean updateParticipantData(int participantDataId, String changedByUser) { org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData participantData = new org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData.Builder() - .withParticipantDataId(dataId) + .withParticipantDataId(participantDataId) .withDdpParticipantId(this.ddpParticipantId) .withDdpInstanceId(this.ddpInstanceId) .withFieldTypeId(this.fieldTypeId) @@ -159,14 +204,29 @@ public boolean updateParticipantData(int dataId, String changedByUser) { .withLastChanged(System.currentTimeMillis()) .withChangedBy(changedByUser) .build(); + int rowsAffected = ((ParticipantDataDao) dataAccess).updateParticipantDataColumn(participantData); + + exportToES(participantDataId, participantData, "participantDataId"); + return rowsAffected == 1; } + private void exportToES(int participantDataId, org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData participantData, String uniqueIdentifier) { + DDPInstanceDto ddpInstanceDto = ((DDPInstanceDao) setDataAccess(new DDPInstanceDao())).getDDPInstanceByInstanceId(ddpInstanceId).orElseThrow(); + + Generator generator = new ParamsGenerator(participantData, ddpInstanceDto.getInstanceName()); + ScriptBuilder scriptBuilder = new NestedScriptBuilder("participantData", uniqueIdentifier); + MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("participantDataId", participantDataId); + NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder(String.join(".", ESObjectConstants.DSM, generator.getPropertyName()), matchQueryBuilder, ScoreMode.Avg); + UpsertPainless upsertPainless = new UpsertPainless(generator, ddpInstanceDto.getEsParticipantIndex(), scriptBuilder, nestedQueryBuilder); + upsertPainless.export(); + } + public Optional findProband(List participantDataList) { return Objects.requireNonNull(participantDataList).stream() .filter(participantDataDto -> { - Map pDataMap = GSON.fromJson(participantDataDto.getData().orElse(""), Map.class); + Map pDataMap = GSON.fromJson(participantDataDto.getData().orElse(StringUtils.EMPTY), Map.class); return FamilyMemberConstants.MEMBER_TYPE_SELF.equals(pDataMap.get(FamilyMemberConstants.MEMBER_TYPE)); }) .findFirst(); @@ -174,22 +234,18 @@ public Optional f public boolean hasFamilyMemberApplicantEmail() { if (Objects.isNull(this.data) || StringUtils.isBlank(this.ddpParticipantId)) return false; - String familyMemberEmail = this.data.get(FamilyMemberConstants.EMAIL); - String esParticipantIndex = new DDPInstanceDao().getEsParticipantIndexByInstanceId(ddpInstanceId).orElse(""); + String familyMemberEmail = this.getData().get(FamilyMemberConstants.EMAIL); + String esParticipantIndex = new DDPInstanceDao().getEsParticipantIndexByInstanceId(ddpInstanceId).orElse(StringUtils.EMPTY); String applicantEmail = ParticipantUtil.getParticipantEmailById(esParticipantIndex, this.ddpParticipantId); return applicantEmail.equalsIgnoreCase(familyMemberEmail); } public boolean hasFamilyMemberApplicantEmail(ESProfile applicantProfile) { if (Objects.isNull(this.data) || StringUtils.isBlank(this.ddpParticipantId)) return false; - String familyMemberEmail = this.data.get(FamilyMemberConstants.EMAIL); - String applicantEmail = StringUtils.defaultIfBlank(applicantProfile.getEmail(), ""); + String familyMemberEmail = this.getData().get(FamilyMemberConstants.EMAIL); + String applicantEmail = StringUtils.defaultIfBlank(applicantProfile.getEmail(), StringUtils.EMPTY); return applicantEmail.equalsIgnoreCase(familyMemberEmail); } - - public String getFamilyMemberEmail() { - return Objects.requireNonNull(this.data).get(FamilyMemberConstants.EMAIL); - } - + } diff --git a/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java b/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java index b9d73c9df..97be4312b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java +++ b/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java @@ -6,6 +6,7 @@ import lombok.NonNull; import org.apache.commons.lang3.StringUtils; +import org.apache.lucene.search.join.ScoreMode; import org.broadinstitute.dsm.db.DDPInstance; import org.broadinstitute.dsm.db.dao.bookmark.BookmarkDao; import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; @@ -17,7 +18,10 @@ import org.broadinstitute.dsm.model.ddp.DDPActivityConstants; import org.broadinstitute.dsm.model.defaultvalues.Defaultable; import org.broadinstitute.dsm.model.elastic.ESActivities; +import org.broadinstitute.dsm.model.elastic.export.generate.Generator; +import org.broadinstitute.dsm.model.elastic.export.painless.NestedScriptBuilder; import org.broadinstitute.dsm.model.elastic.export.painless.ParamsGenerator; +import org.broadinstitute.dsm.model.elastic.export.painless.ScriptBuilder; import org.broadinstitute.dsm.model.elastic.export.painless.UpsertPainless; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; @@ -26,6 +30,8 @@ import org.broadinstitute.dsm.model.settings.field.FieldSettings; import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; +import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.index.query.NestedQueryBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -78,8 +84,14 @@ private boolean extractAndInsertProbandFromESData(DDPInstance instance, ElasticS ); participantData.addDefaultOptionsValueToData(columnsWithDefaultOptions); participantData.insertParticipantData("SYSTEM"); - new ParamsGenerator() - new UpsertPainless(); + + Generator generator = new ParamsGenerator(participantData, instance.getName()); + ScriptBuilder scriptBuilder = new NestedScriptBuilder("participantData", "_id"); + MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("_id", esProfile.getGuid()); + NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder(String.join(".", ESObjectConstants.DSM, generator.getPropertyName()), matchQueryBuilder, ScoreMode.Avg); + UpsertPainless upsertPainless = new UpsertPainless(generator, instance.getParticipantIndexES(), scriptBuilder, nestedQueryBuilder); + upsertPainless.export(); + columnsWithDefaultOptionsFilteredByElasticExportWorkflow.forEach((col, val) -> ElasticSearchUtil.writeWorkflow(WorkflowForES.createInstanceWithStudySpecificData(instance, participantId, col, val, new WorkflowForES.StudySpecificData(probandDataMap.get(FamilyMemberConstants.COLLABORATOR_PARTICIPANT_ID), diff --git a/src/main/java/org/broadinstitute/dsm/util/proxy/jackson/ObjectMapperSingleton.java b/src/main/java/org/broadinstitute/dsm/util/proxy/jackson/ObjectMapperSingleton.java index 406950008..bcd2476c4 100644 --- a/src/main/java/org/broadinstitute/dsm/util/proxy/jackson/ObjectMapperSingleton.java +++ b/src/main/java/org/broadinstitute/dsm/util/proxy/jackson/ObjectMapperSingleton.java @@ -2,6 +2,7 @@ import java.io.IOException; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; public class ObjectMapperSingleton { @@ -12,9 +13,9 @@ public static ObjectMapper instance() { return Helper.objectMapperInstance; } - public static T readValue(String content, Class valueType) { + public static T readValue(String content, TypeReference typeReference) { try { - return Helper.objectMapperInstance.readValue(content, valueType); + return Helper.objectMapperInstance.readValue(content, typeReference); } catch (com.fasterxml.jackson.core.JsonParseException e) { throw new JsonParseException(e.getMessage()); } catch (IOException e) { From adbe1aba7cf0c5c405433bdfa22b962e09df809e Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 28 Jan 2022 16:44:11 +0400 Subject: [PATCH 356/405] DDP-7443 | create facade classes for painless upsert --- .../painless/NestedUpsertPainlessFacade.java | 29 +++++++++++++ .../painless/SingleUpsertPainlessFacade.java | 23 ++++++++++ .../export/painless/UpsertPainlessFacade.java | 43 +++++++++++++++++++ 3 files changed, 95 insertions(+) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedUpsertPainlessFacade.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/SingleUpsertPainlessFacade.java create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/UpsertPainlessFacade.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedUpsertPainlessFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedUpsertPainlessFacade.java new file mode 100644 index 000000000..0f23b9024 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedUpsertPainlessFacade.java @@ -0,0 +1,29 @@ +package org.broadinstitute.dsm.model.elastic.export.painless; + +import org.apache.lucene.search.join.ScoreMode; +import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; +import org.broadinstitute.dsm.statics.DBConstants; +import org.broadinstitute.dsm.statics.ESObjectConstants; +import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.index.query.NestedQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; + +public class NestedUpsertPainlessFacade extends UpsertPainlessFacade { + + NestedUpsertPainlessFacade(Object source, DDPInstanceDto ddpInstanceDto, + String uniqueIdentifier, String fieldName, Object fieldValue) { + super(source, ddpInstanceDto, uniqueIdentifier, fieldName, fieldValue); + } + + @Override + protected ScriptBuilder buildScriptBuilder() { + return new NestedScriptBuilder(generator.getPropertyName(), uniqueIdentifier); + } + + @Override + protected QueryBuilder buildQueryBuilder() { + MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder(fieldName, fieldValue); + return new NestedQueryBuilder(String.join(DBConstants.ALIAS_DELIMITER, ESObjectConstants.DSM, generator.getPropertyName(), + fieldName), matchQueryBuilder, ScoreMode.Avg); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/SingleUpsertPainlessFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/SingleUpsertPainlessFacade.java new file mode 100644 index 000000000..ed3b314a2 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/SingleUpsertPainlessFacade.java @@ -0,0 +1,23 @@ +package org.broadinstitute.dsm.model.elastic.export.painless; + +import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; +import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; + +public class SingleUpsertPainlessFacade extends UpsertPainlessFacade { + + SingleUpsertPainlessFacade(Object source, DDPInstanceDto ddpInstanceDto, + String uniqueIdentifier, String fieldName, Object fieldValue) { + super(source, ddpInstanceDto, uniqueIdentifier, fieldName, fieldValue); + } + + @Override + protected ScriptBuilder buildScriptBuilder() { + return new SingleScriptBuilder(generator.getPropertyName()); + } + + @Override + protected QueryBuilder buildQueryBuilder() { + return new MatchQueryBuilder(fieldName, fieldValue); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/UpsertPainlessFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/UpsertPainlessFacade.java new file mode 100644 index 000000000..60830dd34 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/UpsertPainlessFacade.java @@ -0,0 +1,43 @@ +package org.broadinstitute.dsm.model.elastic.export.painless; + +import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; +import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.model.elastic.export.Exportable; +import org.broadinstitute.dsm.model.elastic.export.generate.BaseGenerator; +import org.broadinstitute.dsm.model.elastic.export.generate.Generator; +import org.elasticsearch.index.query.QueryBuilder; + +public abstract class UpsertPainlessFacade { + + Generator generator; + Exportable upsertPainless; + protected String uniqueIdentifier; + protected String fieldName; + protected Object fieldValue; + + UpsertPainlessFacade(Object source, DDPInstanceDto ddpInstanceDto, String uniqueIdentifier, + String fieldName, Object fieldValue) { + this.uniqueIdentifier = uniqueIdentifier; + this.fieldName = fieldName; + this.fieldValue = fieldValue; + generator = new ParamsGenerator(source, ddpInstanceDto.getInstanceName()); + upsertPainless = new UpsertPainless(generator, ddpInstanceDto.getEsParticipantIndex(), buildScriptBuilder(), buildQueryBuilder()); + } + + protected abstract ScriptBuilder buildScriptBuilder(); + + protected abstract QueryBuilder buildQueryBuilder(); + + public void export() { + upsertPainless.export(); + } + + public static UpsertPainlessFacade of(String alias, Object source, DDPInstanceDto ddpInstanceDto, String uniqueIdentifier, + String fieldName, Object fieldValue) { + BaseGenerator.PropertyInfo propertyInfo = Util.TABLE_ALIAS_MAPPINGS.get(alias); + return propertyInfo.isCollection() + ? new NestedUpsertPainlessFacade(source, ddpInstanceDto, uniqueIdentifier, fieldName, fieldValue) + : new SingleUpsertPainlessFacade(source, ddpInstanceDto, uniqueIdentifier, fieldName, fieldValue); + } + +} From 9e7c03b248f63c2f87882912600ba926289b6b74 Mon Sep 17 00:00:00 2001 From: Nika Date: Fri, 28 Jan 2022 18:51:33 +0400 Subject: [PATCH 357/405] DDP-7443 | use UpsertPainlessFacade.export in RGP and AT cases, refactor code further --- .../dsm/db/KitRequestShipping.java | 38 +++++++++---------- .../org/broadinstitute/dsm/model/BSPKit.java | 6 ++- .../dsm/model/at/DefaultValues.java | 2 +- .../dsm/model/at/ReceiveKitRequest.java | 5 ++- .../model/familymember/AddFamilyMember.java | 8 ++++ .../participant/data/ParticipantData.java | 17 ++++++--- .../rgp/AutomaticProbandDataCreator.java | 21 +++++----- .../dsm/route/KitStatusChangeRoute.java | 17 ++------- .../org/broadinstitute/dsm/util/KitUtil.java | 5 ++- 9 files changed, 65 insertions(+), 54 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java index c78c76073..3447f1316 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java @@ -23,10 +23,7 @@ import org.broadinstitute.dsm.model.ddp.KitDetail; import org.broadinstitute.dsm.model.elastic.export.Exportable; import org.broadinstitute.dsm.model.elastic.export.generate.Generator; -import org.broadinstitute.dsm.model.elastic.export.painless.NestedScriptBuilder; -import org.broadinstitute.dsm.model.elastic.export.painless.ParamsGenerator; -import org.broadinstitute.dsm.model.elastic.export.painless.ScriptBuilder; -import org.broadinstitute.dsm.model.elastic.export.painless.UpsertPainless; +import org.broadinstitute.dsm.model.elastic.export.painless.*; import org.broadinstitute.dsm.statics.ApplicationConfigConstants; import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.statics.ESObjectConstants; @@ -785,7 +782,9 @@ public static void deactivateKitRequest(long dsmKitRequestId, @NonNull String de kitRequestShipping.setDeactivationReason(deactivationReason); kitRequestShipping.setDeactivatedDate(deactivatedDate); - exportToES(kitRequestShipping, ddpInstanceDto, "dsmKitRequestId", "dsmKitRequestId", dsmKitRequestId); + UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, "dsmKitRequestId", "dsmKitRequestId", dsmKitRequestId) + .export(); + } else { if (easypostApiKey != null) { @@ -914,8 +913,9 @@ public static String writeRequest(@NonNull String instanceId, @NonNull String dd DDPInstanceDto ddpInstanceDto = new DDPInstanceDao().getDDPInstanceByInstanceId(Integer.valueOf(ddpInstance.getDdpInstanceId())).orElseThrow(); - exportToES(kitRequestShipping, ddpInstanceDto, "dsmKitRequestId", "_id", Exportable.getParticipantGuid(ddpParticipantId, - ddpInstance.getParticipantIndexES())); + UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, "dsmKitRequestId", "_id", Exportable.getParticipantGuid(ddpParticipantId, ddpInstance.getParticipantIndexES())) + .export(); + } logger.info("Added kitRequest w/ ddpKitRequestId " + ddpKitRequestId); @@ -1002,7 +1002,8 @@ public static void updateKit(long dsmKitId, String userId, DDPInstanceDto ddpIns KitRequestShipping kitRequestShipping = new KitRequestShipping(null, dsmKitId, null, null, null, null); kitRequestShipping.setLabelDate(labelDate); - exportToES(kitRequestShipping, ddpInstanceDto, "dsmKitId", "dsmKitId", dsmKitId); + UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, "dsmKitId", "dsmKitId", dsmKitId) + .export(); } } @@ -1095,7 +1096,10 @@ public static void updateKit(String dsmKitId, Shipment participantShipment, Ship } else { logger.info("Updated kit w/ dsm_kit_id " + dsmKitId, results.resultException); - exportToES(kitRequestShipping, ddpInstanceDto, "dsmKitId", "dsmKitId", dsmKitId); + + UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, "dsmKitId", "dsmKitId", dsmKitId) + .export(); + } } @@ -1390,8 +1394,9 @@ public static void updateKitError(@NonNull long dsmKitRequestId, @NonNull String } KitRequestShipping kitRequestShipping = new KitRequestShipping(dsmKitRequestId, null, null, null, null, message); - exportToES(kitRequestShipping, ddpInstanceDto, "dsmKitRequestId", "dsmKitRequestId", dsmKitRequestId); + UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, "dsmKitRequestId", "dsmKitRequestId", dsmKitRequestId) + .export(); } public static void reactivateKitRequest(@NonNull String kitRequestId, DDPInstanceDto ddpInstanceDto) { @@ -1437,20 +1442,11 @@ public static void reactivateKitRequest(@NonNull String dsmKitRequestId, String long dsmKitId = KitRequestShipping.writeNewKit(dsmKitRequestId, kitRequestShipping.getEasypostAddressId(), message, false); kitRequestShipping.setDsmKitId(dsmKitId); - exportToES(kitRequestShipping, ddpInstanceDto, "dsmKitId", "dsmKitRequestId", dsmKitRequestId); + UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, "dsmKitId", "dsmKitRequestId", dsmKitRequestId) + .export(); } } - public static void exportToES(Object source, DDPInstanceDto ddpInstanceDto, String uniqueIdentifier, - String fieldName, Object fieldValue) { - Generator paramsGenerator = new ParamsGenerator(source, ddpInstanceDto.getInstanceName()); - ScriptBuilder scriptBuilder = new NestedScriptBuilder(paramsGenerator.getPropertyName(), uniqueIdentifier); - MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder(String.join(".",ESObjectConstants.DSM, paramsGenerator.getPropertyName(), fieldName), fieldValue); - NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder(String.join(".",ESObjectConstants.DSM, paramsGenerator.getPropertyName()), matchQueryBuilder, ScoreMode.Avg); - UpsertPainless upsertPainless = new UpsertPainless(paramsGenerator, ddpInstanceDto.getEsParticipantIndex(), scriptBuilder, nestedQueryBuilder); - upsertPainless.export(); - } - public static List findKitRequest(@NonNull String field, @NonNull String value, String[] realms) { Map kitRequests = new HashMap<>(); SimpleResult results = inTransaction((conn) -> { diff --git a/src/main/java/org/broadinstitute/dsm/model/BSPKit.java b/src/main/java/org/broadinstitute/dsm/model/BSPKit.java index 4f92e971e..1f4df273e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/BSPKit.java +++ b/src/main/java/org/broadinstitute/dsm/model/BSPKit.java @@ -14,7 +14,9 @@ import org.broadinstitute.dsm.db.dto.settings.InstanceSettingsDto; import org.broadinstitute.dsm.model.bsp.BSPKitInfo; import org.broadinstitute.dsm.model.bsp.BSPKitStatus; +import org.broadinstitute.dsm.model.elastic.export.painless.UpsertPainlessFacade; import org.broadinstitute.dsm.statics.ApplicationConfigConstants; +import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ElasticSearchDataUtil; import org.broadinstitute.dsm.util.ElasticSearchUtil; @@ -150,7 +152,9 @@ private void updateKitAndExport(String kitLabel, BSPKitDao bspKitDao, BSPKitDto kitRequestShipping.setKitLabel(kitLabel); DDPInstanceDto ddpInstanceDto = new DDPInstanceDao().getDDPInstanceByInstanceName(maybeBspKitQueryResult.getInstanceName()).orElseThrow(); - KitRequestShipping.exportToES(kitRequestShipping, ddpInstanceDto, "kitLabel", "kitLabel", kitLabel); + + UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, "kitLabel", "kitLabel", kitLabel) + .export(); } private void writeSampleReceivedToES(DDPInstance ddpInstance, BSPKitDto bspKitInfo) { diff --git a/src/main/java/org/broadinstitute/dsm/model/at/DefaultValues.java b/src/main/java/org/broadinstitute/dsm/model/at/DefaultValues.java index 77a49afcf..825aae39c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/at/DefaultValues.java +++ b/src/main/java/org/broadinstitute/dsm/model/at/DefaultValues.java @@ -72,7 +72,7 @@ public Map> addDefaultValues() { if (!hasParticipantDataGenomicId(participantDataList) && isSelfOrDependentParticipant(participantDataList)) { ElasticSearchParticipantDto esParticipantData = entry.getValue(); - String hruid = esParticipantData.getProfile().map(ESProfile::getHruid).orElse("");; + String hruid = esParticipantData.getProfile().map(ESProfile::getHruid).orElse(""); addedNewParticipantData = getParticipantGenomicFieldData(participantDataList) .map(pData -> insertGenomicIdIfNotExistsInData(ddpParticipantId, hruid, pData)) .orElseGet(() -> insertGenomicIdForParticipant(ddpParticipantId, hruid)); diff --git a/src/main/java/org/broadinstitute/dsm/model/at/ReceiveKitRequest.java b/src/main/java/org/broadinstitute/dsm/model/at/ReceiveKitRequest.java index 49e28cb4b..ce4f80548 100644 --- a/src/main/java/org/broadinstitute/dsm/model/at/ReceiveKitRequest.java +++ b/src/main/java/org/broadinstitute/dsm/model/at/ReceiveKitRequest.java @@ -9,6 +9,8 @@ import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData; +import org.broadinstitute.dsm.model.elastic.export.painless.UpsertPainlessFacade; +import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.util.NotificationUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -89,7 +91,8 @@ private static boolean updateData(@NonNull String data, @NonNull long participan .withDdpInstanceId(ddpInstanceDto.getDdpInstanceId()) .build(); - KitRequestShipping.exportToES(participantData, ddpInstanceDto, "participantDataId", "participantDataId", participantDataId); + UpsertPainlessFacade.of(DBConstants.DDP_PARTICIPANT_DATA_ALIAS, participantData, ddpInstanceDto, "participantDataId", "participantDataId", participantDataId) + .export(); return true; } diff --git a/src/main/java/org/broadinstitute/dsm/model/familymember/AddFamilyMember.java b/src/main/java/org/broadinstitute/dsm/model/familymember/AddFamilyMember.java index d6a9c40ab..835ce7405 100644 --- a/src/main/java/org/broadinstitute/dsm/model/familymember/AddFamilyMember.java +++ b/src/main/java/org/broadinstitute/dsm/model/familymember/AddFamilyMember.java @@ -9,6 +9,7 @@ import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.DDPInstance; +import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; import org.broadinstitute.dsm.db.dto.user.UserDto; import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; import org.broadinstitute.dsm.db.dao.settings.FieldSettingsDao; @@ -17,11 +18,14 @@ import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; import org.broadinstitute.dsm.export.WorkflowForES; import org.broadinstitute.dsm.model.Study; +import org.broadinstitute.dsm.model.elastic.export.Exportable; +import org.broadinstitute.dsm.model.elastic.export.painless.UpsertPainlessFacade; import org.broadinstitute.dsm.model.participant.data.AddFamilyMemberPayload; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import org.broadinstitute.dsm.model.participant.data.FamilyMemberDetails; import org.broadinstitute.dsm.model.rgp.RgpAddFamilyMember; import org.broadinstitute.dsm.model.settings.field.FieldSettings; +import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.broadinstitute.dsm.util.ParticipantUtil; import org.slf4j.Logger; @@ -95,6 +99,10 @@ public void exportDataToEs() { } else { exportDefaultWorkflowsForFamilyMemberToES(); } + DDPInstanceDto ddpInstanceDto = ddpInstanceDao.getDDPInstanceByGuid(studyGuid).orElseThrow(); + String participantGuid = Exportable.getParticipantGuid(ddpParticipantId, ddpInstanceDto.getEsParticipantIndex()); + UpsertPainlessFacade.of(DBConstants.DDP_PARTICIPANT_DATA_ALIAS, participantData, ddpInstanceDto, "participantDataId", "_id", participantGuid) + .export(); } protected void exportProbandDataForFamilyMemberToEs() { diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java b/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java index bcd0a77a3..6a4aff077 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java @@ -25,11 +25,9 @@ import org.broadinstitute.dsm.db.structure.ColumnName; import org.broadinstitute.dsm.db.structure.TableName; import org.broadinstitute.dsm.model.elastic.ESProfile; +import org.broadinstitute.dsm.model.elastic.export.Exportable; import org.broadinstitute.dsm.model.elastic.export.generate.Generator; -import org.broadinstitute.dsm.model.elastic.export.painless.NestedScriptBuilder; -import org.broadinstitute.dsm.model.elastic.export.painless.ParamsGenerator; -import org.broadinstitute.dsm.model.elastic.export.painless.ScriptBuilder; -import org.broadinstitute.dsm.model.elastic.export.painless.UpsertPainless; +import org.broadinstitute.dsm.model.elastic.export.painless.*; import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.proxy.jackson.ObjectMapperSingleton; @@ -170,7 +168,11 @@ public long insertParticipantData(String userEmail) { } logger.info("Successfully inserted data for participant: " + this.ddpParticipantId); - exportToES("participantDataId"); + DDPInstanceDto ddpInstanceDto = new DDPInstanceDao().getDDPInstanceByInstanceId(ddpInstanceId).orElseThrow(); + String participantGuid = Exportable.getParticipantGuid(ddpParticipantId, ddpInstanceDto.getEsParticipantIndex()); + + UpsertPainlessFacade.of(DBConstants.DDP_PARTICIPANT_DATA_ALIAS, participantData, ddpInstanceDto, "participantDataId", "_id", participantGuid) + .export(); return createdDataKey; } @@ -207,7 +209,10 @@ public boolean updateParticipantData(int participantDataId, String changedByUser int rowsAffected = ((ParticipantDataDao) dataAccess).updateParticipantDataColumn(participantData); - exportToES(participantDataId, participantData, "participantDataId"); + DDPInstanceDto ddpInstanceDto = new DDPInstanceDao().getDDPInstanceByInstanceId(ddpInstanceId).orElseThrow(); + + UpsertPainlessFacade.of(DBConstants.DDP_PARTICIPANT_DATA_ALIAS, participantData, ddpInstanceDto, "participantDataId", "participantDataId", participantDataId) + .export(); return rowsAffected == 1; } diff --git a/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java b/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java index 97be4312b..e2361ea1a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java +++ b/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java @@ -7,27 +7,27 @@ import lombok.NonNull; import org.apache.commons.lang3.StringUtils; import org.apache.lucene.search.join.ScoreMode; +import org.apache.lucene.util.automaton.DaciukMihovAutomatonBuilder; import org.broadinstitute.dsm.db.DDPInstance; import org.broadinstitute.dsm.db.dao.bookmark.BookmarkDao; import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; import org.broadinstitute.dsm.db.dao.settings.FieldSettingsDao; import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; import org.broadinstitute.dsm.db.dto.bookmark.BookmarkDto; +import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; import org.broadinstitute.dsm.export.WorkflowForES; import org.broadinstitute.dsm.model.ddp.DDPActivityConstants; import org.broadinstitute.dsm.model.defaultvalues.Defaultable; import org.broadinstitute.dsm.model.elastic.ESActivities; import org.broadinstitute.dsm.model.elastic.export.generate.Generator; -import org.broadinstitute.dsm.model.elastic.export.painless.NestedScriptBuilder; -import org.broadinstitute.dsm.model.elastic.export.painless.ParamsGenerator; -import org.broadinstitute.dsm.model.elastic.export.painless.ScriptBuilder; -import org.broadinstitute.dsm.model.elastic.export.painless.UpsertPainless; +import org.broadinstitute.dsm.model.elastic.export.painless.*; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import org.broadinstitute.dsm.model.participant.data.FamilyMemberDetails; import org.broadinstitute.dsm.model.participant.data.ParticipantData; import org.broadinstitute.dsm.model.settings.field.FieldSettings; +import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.elasticsearch.index.query.MatchQueryBuilder; @@ -85,12 +85,13 @@ private boolean extractAndInsertProbandFromESData(DDPInstance instance, ElasticS participantData.addDefaultOptionsValueToData(columnsWithDefaultOptions); participantData.insertParticipantData("SYSTEM"); - Generator generator = new ParamsGenerator(participantData, instance.getName()); - ScriptBuilder scriptBuilder = new NestedScriptBuilder("participantData", "_id"); - MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("_id", esProfile.getGuid()); - NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder(String.join(".", ESObjectConstants.DSM, generator.getPropertyName()), matchQueryBuilder, ScoreMode.Avg); - UpsertPainless upsertPainless = new UpsertPainless(generator, instance.getParticipantIndexES(), scriptBuilder, nestedQueryBuilder); - upsertPainless.export(); + DDPInstanceDto ddpInstanceDto = new DDPInstanceDto.Builder() + .withInstanceName(instance.getName()) + .withEsParticipantIndex(instance.getParticipantIndexES()) + .build(); + + UpsertPainlessFacade.of(DBConstants.DDP_PARTICIPANT_DATA_ALIAS, participantData, ddpInstanceDto, "participantDataId", "_id", esProfile.getGuid()) + .export(); columnsWithDefaultOptionsFilteredByElasticExportWorkflow.forEach((col, val) -> ElasticSearchUtil.writeWorkflow(WorkflowForES.createInstanceWithStudySpecificData(instance, participantId, col, val, diff --git a/src/main/java/org/broadinstitute/dsm/route/KitStatusChangeRoute.java b/src/main/java/org/broadinstitute/dsm/route/KitStatusChangeRoute.java index 5664dffc1..5acb2cfb7 100644 --- a/src/main/java/org/broadinstitute/dsm/route/KitStatusChangeRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/KitStatusChangeRoute.java @@ -16,6 +16,7 @@ import org.broadinstitute.dsm.model.KitDDPNotification; import org.broadinstitute.dsm.model.KitRequestSettings; import org.broadinstitute.dsm.model.at.ReceiveKitRequest; +import org.broadinstitute.dsm.model.elastic.export.painless.UpsertPainlessFacade; import org.broadinstitute.dsm.security.RequestHandler; import org.broadinstitute.dsm.statics.*; import org.broadinstitute.dsm.util.*; @@ -184,11 +185,12 @@ else if (RoutePath.RECEIVED_KIT_REQUEST.equals(changeType)) { if (kitDDPNotification != null) { EventUtil.triggerDDP(conn, kitDDPNotification); } - KitRequestShipping.exportToES(kitRequestShipping, ddpInstanceDto, "ddpLabel", "ddpLabel", kit); + UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, "ddpLabel", "ddpLabel", kit); } else if (RoutePath.TRACKING_SCAN_REQUEST.equals(changeType)) { logger.info("Added tracking for kit w/ kit_label " + kit); - KitRequestShipping.exportToES(kitRequestShipping, ddpInstanceDto, "kitLabel", "kitLabel", addValue); + UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, "kitLabel", "kitLabel", addValue) + .export(); } else if (RoutePath.RECEIVED_KIT_REQUEST.equals(changeType)) { logger.info("Updated kitRequest w/ SM-ID kit_label " + kit); @@ -240,17 +242,6 @@ else if (RoutePath.RECEIVED_KIT_REQUEST.equals(changeType)) { else { logger.error("Error something went wrong at the scan pages"); } - } else { - if (Objects.isNull(ddpInstanceDto)) return; - if (RoutePath.FINAL_SCAN_REQUEST.equals(changeType) || RoutePath.SENT_KIT_REQUEST.equals(changeType)) { - KitRequestShipping.exportToES(kitRequestShipping, ddpInstanceDto, "ddpLabel", "ddpLabel", kit); - } - else if (RoutePath.TRACKING_SCAN_REQUEST.equals(changeType)) { - KitRequestShipping.exportToES(kitRequestShipping, ddpInstanceDto, "kitLabel", "kitLable", kit); - } - else if (RoutePath.RECEIVED_KIT_REQUEST.equals(changeType)) { - //TODO - sm_id - } } } diff --git a/src/main/java/org/broadinstitute/dsm/util/KitUtil.java b/src/main/java/org/broadinstitute/dsm/util/KitUtil.java index 33c46f9d7..1a9a3be12 100644 --- a/src/main/java/org/broadinstitute/dsm/util/KitUtil.java +++ b/src/main/java/org/broadinstitute/dsm/util/KitUtil.java @@ -19,6 +19,7 @@ import org.broadinstitute.dsm.model.KitType; import org.broadinstitute.dsm.model.Value; import org.broadinstitute.dsm.model.ddp.DDPParticipant; +import org.broadinstitute.dsm.model.elastic.export.painless.UpsertPainlessFacade; import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.statics.QueryExtension; import org.slf4j.Logger; @@ -511,7 +512,9 @@ private static SimpleResult updateKit(@NonNull Connection conn, String status, L kitRequestShipping.setEasypostShipmentStatus(status); kitRequestShipping.setMessage(message); kitRequestShipping.setDsmKitId(dsmKitId); - KitRequestShipping.exportToES(kitRequestShipping, ddpInstanceDto, "dsmKitId", "dsmKitId", dsmKitId); + + UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, "dsmKitId", "dsmKitId", dsmKitId) + .export(); return dbVals; } From 0b89ff2e037360b40454d0cfe1ea28a8f6a09883 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 31 Jan 2022 10:50:29 +0400 Subject: [PATCH 358/405] DDP-7443 | extract basic default maker class --- .../dsm/model/BasicDefaultDataMaker.java | 53 +++++++++++++++++++ .../rgp/AutomaticProbandDataCreator.java | 36 ++----------- 2 files changed, 57 insertions(+), 32 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/BasicDefaultDataMaker.java diff --git a/src/main/java/org/broadinstitute/dsm/model/BasicDefaultDataMaker.java b/src/main/java/org/broadinstitute/dsm/model/BasicDefaultDataMaker.java new file mode 100644 index 000000000..5852eb7aa --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/BasicDefaultDataMaker.java @@ -0,0 +1,53 @@ +package org.broadinstitute.dsm.model; + +import java.util.List; +import java.util.Optional; + +import org.broadinstitute.dsm.db.DDPInstance; +import org.broadinstitute.dsm.db.dao.bookmark.BookmarkDao; +import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; +import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; +import org.broadinstitute.dsm.db.dao.settings.FieldSettingsDao; +import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; +import org.broadinstitute.dsm.model.defaultvalues.Defaultable; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; +import org.broadinstitute.dsm.model.rgp.AutomaticProbandDataCreator; +import org.broadinstitute.dsm.model.settings.field.FieldSettings; +import org.broadinstitute.dsm.util.ElasticSearchUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class BasicDefaultDataMaker implements Defaultable { + protected static final Logger logger = LoggerFactory.getLogger(AutomaticProbandDataCreator.class); + private final FieldSettings fieldSettings = new FieldSettings(); + private final BookmarkDao bookmarkDao = new BookmarkDao(); + private final ParticipantDataDao participantDataDao = new ParticipantDataDao(); + private final DDPInstanceDao ddpInstanceDao = new DDPInstanceDao(); + protected DDPInstance instance; + + private boolean setDefaultProbandData(Optional maybeParticipantESData) { + if (maybeParticipantESData.isEmpty()) { + logger.warn("Could not create proband/self data, participant ES data is null"); + return false; + } + List fieldSettingsDtosByOptionAndInstanceId = + FieldSettingsDao.of().getOptionAndRadioFieldSettingsByInstanceId(Integer.parseInt(instance.getDdpInstanceId())); + + return maybeParticipantESData + .map(elasticSearch -> extractAndInsertProbandFromESData(instance, elasticSearch, fieldSettingsDtosByOptionAndInstanceId)) + .orElse(false); + } + + protected abstract boolean extractAndInsertProbandFromESData(DDPInstance instance, ElasticSearchParticipantDto esData, + List fieldSettingsDtosByOptionAndInstanceId); + + @Override + public boolean generateDefaults(String studyGuid, String participantId) { + String esParticipantIndex = ddpInstanceDao.getEsParticipantIndexByStudyGuid(studyGuid) + .orElse(""); + Optional maybeParticipantESDataByParticipantId = + ElasticSearchUtil.getParticipantESDataByParticipantId(esParticipantIndex, participantId); + instance = DDPInstance.getDDPInstance(studyGuid); + return setDefaultProbandData(maybeParticipantESDataByParticipantId); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java b/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java index e2361ea1a..db4bcf06c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java +++ b/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java @@ -35,33 +35,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class AutomaticProbandDataCreator implements Defaultable { +public class AutomaticProbandDataCreator extends org.broadinstitute.dsm.model.BasicDefaultDataMaker { - private static final Logger logger = LoggerFactory.getLogger(AutomaticProbandDataCreator.class); public static final String RGP_FAMILY_ID = "rgp_family_id"; - private final FieldSettings fieldSettings = new FieldSettings(); - private final BookmarkDao bookmarkDao = new BookmarkDao(); - private final ParticipantDataDao participantDataDao = new ParticipantDataDao(); - private final DDPInstanceDao ddpInstanceDao = new DDPInstanceDao(); - private DDPInstance instance; - - private boolean setDefaultProbandData(Optional maybeParticipantESData) { - if (maybeParticipantESData.isEmpty()) { - logger.warn("Could not create proband/self data, participant ES data is null"); - return false; - } - List fieldSettingsDtosByOptionAndInstanceId = - FieldSettingsDao.of().getOptionAndRadioFieldSettingsByInstanceId(Integer.parseInt(instance.getDdpInstanceId())); - - return maybeParticipantESData - .map(elasticSearch -> extractAndInsertProbandFromESData(instance, elasticSearch, fieldSettingsDtosByOptionAndInstanceId)) - .orElse(false); - } - - private boolean extractAndInsertProbandFromESData(DDPInstance instance, ElasticSearchParticipantDto esData, - List fieldSettingsDtosByOptionAndInstanceId) { + @Override + protected boolean extractAndInsertProbandFromESData(DDPInstance instance, ElasticSearchParticipantDto esData, + List fieldSettingsDtosByOptionAndInstanceId) { return esData.getProfile() .map(esProfile -> { @@ -166,13 +147,4 @@ void insertFamilyIdToDsmES(@NonNull String esIndex, @NonNull String participantI } } - @Override - public boolean generateDefaults(String studyGuid, String participantId) { - String esParticipantIndex = ddpInstanceDao.getEsParticipantIndexByStudyGuid(studyGuid) - .orElse(""); - Optional maybeParticipantESDataByParticipantId = - ElasticSearchUtil.getParticipantESDataByParticipantId(esParticipantIndex, participantId); - instance = DDPInstance.getDDPInstance(studyGuid); - return setDefaultProbandData(maybeParticipantESDataByParticipantId); - } } From de25ded9dfab2b372b41a60e7c46173f464353db Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 31 Jan 2022 11:20:25 +0400 Subject: [PATCH 359/405] DDP-7443 | remove unused abstract method, make it shorter --- .../dsm/model/BasicDefaultDataMaker.java | 38 +++++++------------ .../rgp/AutomaticProbandDataCreator.java | 6 ++- 2 files changed, 18 insertions(+), 26 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/BasicDefaultDataMaker.java b/src/main/java/org/broadinstitute/dsm/model/BasicDefaultDataMaker.java index 5852eb7aa..16a5ebff8 100644 --- a/src/main/java/org/broadinstitute/dsm/model/BasicDefaultDataMaker.java +++ b/src/main/java/org/broadinstitute/dsm/model/BasicDefaultDataMaker.java @@ -1,14 +1,9 @@ package org.broadinstitute.dsm.model; -import java.util.List; -import java.util.Optional; - import org.broadinstitute.dsm.db.DDPInstance; import org.broadinstitute.dsm.db.dao.bookmark.BookmarkDao; import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; -import org.broadinstitute.dsm.db.dao.settings.FieldSettingsDao; -import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; import org.broadinstitute.dsm.model.defaultvalues.Defaultable; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.rgp.AutomaticProbandDataCreator; @@ -17,29 +12,20 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Optional; + public abstract class BasicDefaultDataMaker implements Defaultable { + protected static final Logger logger = LoggerFactory.getLogger(AutomaticProbandDataCreator.class); - private final FieldSettings fieldSettings = new FieldSettings(); - private final BookmarkDao bookmarkDao = new BookmarkDao(); - private final ParticipantDataDao participantDataDao = new ParticipantDataDao(); - private final DDPInstanceDao ddpInstanceDao = new DDPInstanceDao(); - protected DDPInstance instance; - private boolean setDefaultProbandData(Optional maybeParticipantESData) { - if (maybeParticipantESData.isEmpty()) { - logger.warn("Could not create proband/self data, participant ES data is null"); - return false; - } - List fieldSettingsDtosByOptionAndInstanceId = - FieldSettingsDao.of().getOptionAndRadioFieldSettingsByInstanceId(Integer.parseInt(instance.getDdpInstanceId())); + protected final FieldSettings fieldSettings = new FieldSettings(); + protected final BookmarkDao bookmarkDao = new BookmarkDao(); + protected final ParticipantDataDao participantDataDao = new ParticipantDataDao(); + protected final DDPInstanceDao ddpInstanceDao = new DDPInstanceDao(); + protected DDPInstance instance; - return maybeParticipantESData - .map(elasticSearch -> extractAndInsertProbandFromESData(instance, elasticSearch, fieldSettingsDtosByOptionAndInstanceId)) - .orElse(false); - } - protected abstract boolean extractAndInsertProbandFromESData(DDPInstance instance, ElasticSearchParticipantDto esData, - List fieldSettingsDtosByOptionAndInstanceId); + protected abstract boolean setDefaultData(ElasticSearchParticipantDto esData); @Override public boolean generateDefaults(String studyGuid, String participantId) { @@ -47,7 +33,11 @@ public boolean generateDefaults(String studyGuid, String participantId) { .orElse(""); Optional maybeParticipantESDataByParticipantId = ElasticSearchUtil.getParticipantESDataByParticipantId(esParticipantIndex, participantId); + if (maybeParticipantESDataByParticipantId.isEmpty()) { + logger.warn("Could not create proband/self data, participant ES data is null"); + return false; + } instance = DDPInstance.getDDPInstance(studyGuid); - return setDefaultProbandData(maybeParticipantESDataByParticipantId); + return setDefaultData(maybeParticipantESDataByParticipantId.get()); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java b/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java index db4bcf06c..eb20f1ad2 100644 --- a/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java +++ b/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java @@ -41,8 +41,10 @@ public class AutomaticProbandDataCreator extends org.broadinstitute.dsm.model.Ba public static final String RGP_FAMILY_ID = "rgp_family_id"; @Override - protected boolean extractAndInsertProbandFromESData(DDPInstance instance, ElasticSearchParticipantDto esData, - List fieldSettingsDtosByOptionAndInstanceId) { + protected boolean setDefaultData(ElasticSearchParticipantDto esData) { + + List fieldSettingsDtosByOptionAndInstanceId = + FieldSettingsDao.of().getOptionAndRadioFieldSettingsByInstanceId(Integer.parseInt(instance.getDdpInstanceId())); return esData.getProfile() .map(esProfile -> { From 79fb2d7fcd1d2663e067121681c847caa02340be Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 31 Jan 2022 11:50:44 +0400 Subject: [PATCH 360/405] DDP-7443 | organization changes --- .../dsm/model/at/DefaultValues.java | 35 ++++++++++++++----- .../BasicDefaultDataMaker.java | 9 +++-- .../model/defaultvalues/DefaultableMaker.java | 2 +- .../rgp/AutomaticProbandDataCreator.java | 18 ++-------- .../search/ElasticSearchParticipantDto.java | 2 +- .../model/familymember/AddFamilyMember.java | 2 +- .../rgp/RgpAddFamilyMember.java | 2 +- .../familymember/AddFamilyMemberRoute.java | 2 -- .../rgp/AutomaticProbandDataCreatorTest.java | 2 +- 9 files changed, 38 insertions(+), 36 deletions(-) rename src/main/java/org/broadinstitute/dsm/model/{ => defaultvalues}/BasicDefaultDataMaker.java (91%) rename src/main/java/org/broadinstitute/dsm/model/{ => defaultvalues}/rgp/AutomaticProbandDataCreator.java (90%) rename src/main/java/org/broadinstitute/dsm/model/{ => familymember}/rgp/RgpAddFamilyMember.java (94%) diff --git a/src/main/java/org/broadinstitute/dsm/model/at/DefaultValues.java b/src/main/java/org/broadinstitute/dsm/model/at/DefaultValues.java index 825aae39c..82ce02557 100644 --- a/src/main/java/org/broadinstitute/dsm/model/at/DefaultValues.java +++ b/src/main/java/org/broadinstitute/dsm/model/at/DefaultValues.java @@ -12,6 +12,7 @@ import org.broadinstitute.dsm.db.dto.bookmark.BookmarkDto; import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData; import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; +import org.broadinstitute.dsm.model.defaultvalues.BasicDefaultDataMaker; import org.broadinstitute.dsm.model.elastic.ESProfile; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.settings.field.FieldSettings; @@ -25,7 +26,7 @@ import java.util.function.Function; import java.util.stream.Collectors; -public class DefaultValues { +public class DefaultValues extends BasicDefaultDataMaker { private static final Logger logger = LoggerFactory.getLogger(DefaultValues.class); @@ -48,14 +49,15 @@ public class DefaultValues { private String queryAddition; private ParticipantDataDao participantDataDao; - public DefaultValues(Map> participantData, - List participantESData, @NonNull DDPInstance instance, String queryAddition) { - this.participantData = participantData; - this.participantESData = participantESData; - this.instance = instance; - this.queryAddition = queryAddition; - this.participantDataDao = new ParticipantDataDao(); - } +// +// public DefaultValues(Map> participantData, +// List participantESData, @NonNull DDPInstance instance, String queryAddition) { +// this.participantData = participantData; +// this.participantESData = participantESData; +// this.instance = instance; +// this.queryAddition = queryAddition; +// this.participantDataDao = new ParticipantDataDao(); +// } public Map> addDefaultValues() { if (participantESData == null) { @@ -230,4 +232,19 @@ private String getDefaultExitStatus() { private void setDataAccess(Dao dao) { this.dataAccess = dao; } + + @Override + protected boolean setDefaultData(ElasticSearchParticipantDto esData) { + isSelfOrDependentParticipant() + return esData.getProfile() + .map(esProfile -> { + return + insertGenomicIdForParticipant(ddpParticipantId, hruid); + insertExitStatusForParticipant(ddpParticipantId); + }) + .orElseGet(() -> { + logger.info("Participant does not have ES profile yet..."); + return false; + }); + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/BasicDefaultDataMaker.java b/src/main/java/org/broadinstitute/dsm/model/defaultvalues/BasicDefaultDataMaker.java similarity index 91% rename from src/main/java/org/broadinstitute/dsm/model/BasicDefaultDataMaker.java rename to src/main/java/org/broadinstitute/dsm/model/defaultvalues/BasicDefaultDataMaker.java index 16a5ebff8..944160517 100644 --- a/src/main/java/org/broadinstitute/dsm/model/BasicDefaultDataMaker.java +++ b/src/main/java/org/broadinstitute/dsm/model/defaultvalues/BasicDefaultDataMaker.java @@ -1,19 +1,18 @@ -package org.broadinstitute.dsm.model; +package org.broadinstitute.dsm.model.defaultvalues; + +import java.util.Optional; import org.broadinstitute.dsm.db.DDPInstance; import org.broadinstitute.dsm.db.dao.bookmark.BookmarkDao; import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; -import org.broadinstitute.dsm.model.defaultvalues.Defaultable; +import org.broadinstitute.dsm.model.defaultvalues.rgp.AutomaticProbandDataCreator; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; -import org.broadinstitute.dsm.model.rgp.AutomaticProbandDataCreator; import org.broadinstitute.dsm.model.settings.field.FieldSettings; import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Optional; - public abstract class BasicDefaultDataMaker implements Defaultable { protected static final Logger logger = LoggerFactory.getLogger(AutomaticProbandDataCreator.class); diff --git a/src/main/java/org/broadinstitute/dsm/model/defaultvalues/DefaultableMaker.java b/src/main/java/org/broadinstitute/dsm/model/defaultvalues/DefaultableMaker.java index 88b0a3863..d3ae0560d 100644 --- a/src/main/java/org/broadinstitute/dsm/model/defaultvalues/DefaultableMaker.java +++ b/src/main/java/org/broadinstitute/dsm/model/defaultvalues/DefaultableMaker.java @@ -1,7 +1,7 @@ package org.broadinstitute.dsm.model.defaultvalues; import org.broadinstitute.dsm.model.Study; -import org.broadinstitute.dsm.model.rgp.AutomaticProbandDataCreator; +import org.broadinstitute.dsm.model.defaultvalues.rgp.AutomaticProbandDataCreator; public class DefaultableMaker { diff --git a/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java b/src/main/java/org/broadinstitute/dsm/model/defaultvalues/rgp/AutomaticProbandDataCreator.java similarity index 90% rename from src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java rename to src/main/java/org/broadinstitute/dsm/model/defaultvalues/rgp/AutomaticProbandDataCreator.java index eb20f1ad2..842c2283a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreator.java +++ b/src/main/java/org/broadinstitute/dsm/model/defaultvalues/rgp/AutomaticProbandDataCreator.java @@ -1,4 +1,4 @@ -package org.broadinstitute.dsm.model.rgp; +package org.broadinstitute.dsm.model.defaultvalues.rgp; import java.util.List; import java.util.Map; @@ -6,36 +6,24 @@ import lombok.NonNull; import org.apache.commons.lang3.StringUtils; -import org.apache.lucene.search.join.ScoreMode; -import org.apache.lucene.util.automaton.DaciukMihovAutomatonBuilder; -import org.broadinstitute.dsm.db.DDPInstance; -import org.broadinstitute.dsm.db.dao.bookmark.BookmarkDao; -import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; import org.broadinstitute.dsm.db.dao.settings.FieldSettingsDao; -import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; import org.broadinstitute.dsm.db.dto.bookmark.BookmarkDto; import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; import org.broadinstitute.dsm.export.WorkflowForES; import org.broadinstitute.dsm.model.ddp.DDPActivityConstants; -import org.broadinstitute.dsm.model.defaultvalues.Defaultable; +import org.broadinstitute.dsm.model.defaultvalues.BasicDefaultDataMaker; import org.broadinstitute.dsm.model.elastic.ESActivities; -import org.broadinstitute.dsm.model.elastic.export.generate.Generator; import org.broadinstitute.dsm.model.elastic.export.painless.*; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import org.broadinstitute.dsm.model.participant.data.FamilyMemberDetails; import org.broadinstitute.dsm.model.participant.data.ParticipantData; -import org.broadinstitute.dsm.model.settings.field.FieldSettings; import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; -import org.elasticsearch.index.query.MatchQueryBuilder; -import org.elasticsearch.index.query.NestedQueryBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -public class AutomaticProbandDataCreator extends org.broadinstitute.dsm.model.BasicDefaultDataMaker { +public class AutomaticProbandDataCreator extends BasicDefaultDataMaker { public static final String RGP_FAMILY_ID = "rgp_family_id"; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchParticipantDto.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchParticipantDto.java index 35ede491b..4136f536a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchParticipantDto.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/ElasticSearchParticipantDto.java @@ -93,7 +93,7 @@ public String getParticipantId() { return getProfile().map(esProfile -> StringUtils.isNotBlank(esProfile.getGuid()) ? esProfile.getGuid() : esProfile.getLegacyAltPid()) - .orElse(""); + .orElse(StringUtils.EMPTY); } public ElasticSearchParticipantDto() {} diff --git a/src/main/java/org/broadinstitute/dsm/model/familymember/AddFamilyMember.java b/src/main/java/org/broadinstitute/dsm/model/familymember/AddFamilyMember.java index 835ce7405..5974f1449 100644 --- a/src/main/java/org/broadinstitute/dsm/model/familymember/AddFamilyMember.java +++ b/src/main/java/org/broadinstitute/dsm/model/familymember/AddFamilyMember.java @@ -23,7 +23,7 @@ import org.broadinstitute.dsm.model.participant.data.AddFamilyMemberPayload; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import org.broadinstitute.dsm.model.participant.data.FamilyMemberDetails; -import org.broadinstitute.dsm.model.rgp.RgpAddFamilyMember; +import org.broadinstitute.dsm.model.familymember.rgp.RgpAddFamilyMember; import org.broadinstitute.dsm.model.settings.field.FieldSettings; import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; diff --git a/src/main/java/org/broadinstitute/dsm/model/rgp/RgpAddFamilyMember.java b/src/main/java/org/broadinstitute/dsm/model/familymember/rgp/RgpAddFamilyMember.java similarity index 94% rename from src/main/java/org/broadinstitute/dsm/model/rgp/RgpAddFamilyMember.java rename to src/main/java/org/broadinstitute/dsm/model/familymember/rgp/RgpAddFamilyMember.java index 8bd855362..d2c071a52 100644 --- a/src/main/java/org/broadinstitute/dsm/model/rgp/RgpAddFamilyMember.java +++ b/src/main/java/org/broadinstitute/dsm/model/familymember/rgp/RgpAddFamilyMember.java @@ -1,4 +1,4 @@ -package org.broadinstitute.dsm.model.rgp; +package org.broadinstitute.dsm.model.familymember.rgp; import org.broadinstitute.dsm.model.familymember.AddFamilyMember; import org.broadinstitute.dsm.model.participant.data.AddFamilyMemberPayload; diff --git a/src/main/java/org/broadinstitute/dsm/route/familymember/AddFamilyMemberRoute.java b/src/main/java/org/broadinstitute/dsm/route/familymember/AddFamilyMemberRoute.java index 51dd723a7..aadf12823 100644 --- a/src/main/java/org/broadinstitute/dsm/route/familymember/AddFamilyMemberRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/familymember/AddFamilyMemberRoute.java @@ -5,7 +5,6 @@ import com.google.gson.Gson; import org.broadinstitute.ddp.handlers.util.Result; -import org.broadinstitute.dsm.db.DDPInstance; import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; import org.broadinstitute.dsm.model.Study; @@ -13,7 +12,6 @@ import org.broadinstitute.dsm.model.participant.data.AddFamilyMemberPayload; import org.broadinstitute.dsm.model.participant.data.FamilyMemberDetails; import org.broadinstitute.dsm.model.participant.data.ParticipantData; -import org.broadinstitute.dsm.model.rgp.RgpAddFamilyMember; import org.broadinstitute.dsm.security.RequestHandler; import org.broadinstitute.dsm.statics.DBConstants; import org.slf4j.Logger; diff --git a/src/test/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreatorTest.java b/src/test/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreatorTest.java index affd4a131..4ee536823 100644 --- a/src/test/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreatorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreatorTest.java @@ -1,10 +1,10 @@ package org.broadinstitute.dsm.model.rgp; import static org.broadinstitute.dsm.TestHelper.setupDB; -import static org.junit.Assert.*; import java.util.Map; +import org.broadinstitute.dsm.model.defaultvalues.rgp.AutomaticProbandDataCreator; import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.junit.Assert; From 2d2e1385a48ecaa993b888ca4a8866986a08777e Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 31 Jan 2022 12:31:33 +0400 Subject: [PATCH 361/405] DDP-7443 | move (AT,RGP) export logic under "defaultvalues" package, start writing test for some predicates --- .../defaultvalues/BasicDefaultDataMaker.java | 10 +++--- .../{at => defaultvalues}/DefaultValues.java | 15 ++++---- .../model/defaultvalues/DefaultableMaker.java | 3 +- ...va => RgpAutomaticProbandDataCreator.java} | 4 +-- .../model/participant/ParticipantWrapper.java | 2 +- .../defaultvalues/DefaultValuesTest.java | 36 +++++++++++++++++++ .../RgpAutomaticProbandDataCreatorTest.java} | 8 ++--- 7 files changed, 58 insertions(+), 20 deletions(-) rename src/main/java/org/broadinstitute/dsm/model/{at => defaultvalues}/DefaultValues.java (96%) rename src/main/java/org/broadinstitute/dsm/model/defaultvalues/{rgp/AutomaticProbandDataCreator.java => RgpAutomaticProbandDataCreator.java} (98%) create mode 100644 src/test/java/org/broadinstitute/dsm/model/defaultvalues/DefaultValuesTest.java rename src/test/java/org/broadinstitute/dsm/model/{rgp/AutomaticProbandDataCreatorTest.java => defaultvalues/RgpAutomaticProbandDataCreatorTest.java} (79%) diff --git a/src/main/java/org/broadinstitute/dsm/model/defaultvalues/BasicDefaultDataMaker.java b/src/main/java/org/broadinstitute/dsm/model/defaultvalues/BasicDefaultDataMaker.java index 944160517..da1caedf7 100644 --- a/src/main/java/org/broadinstitute/dsm/model/defaultvalues/BasicDefaultDataMaker.java +++ b/src/main/java/org/broadinstitute/dsm/model/defaultvalues/BasicDefaultDataMaker.java @@ -6,7 +6,6 @@ import org.broadinstitute.dsm.db.dao.bookmark.BookmarkDao; import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; -import org.broadinstitute.dsm.model.defaultvalues.rgp.AutomaticProbandDataCreator; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.settings.field.FieldSettings; import org.broadinstitute.dsm.util.ElasticSearchUtil; @@ -15,16 +14,18 @@ public abstract class BasicDefaultDataMaker implements Defaultable { - protected static final Logger logger = LoggerFactory.getLogger(AutomaticProbandDataCreator.class); + protected static final Logger logger = LoggerFactory.getLogger(RgpAutomaticProbandDataCreator.class); protected final FieldSettings fieldSettings = new FieldSettings(); protected final BookmarkDao bookmarkDao = new BookmarkDao(); protected final ParticipantDataDao participantDataDao = new ParticipantDataDao(); protected final DDPInstanceDao ddpInstanceDao = new DDPInstanceDao(); protected DDPInstance instance; + protected ElasticSearchParticipantDto elasticSearchParticipantDto; - protected abstract boolean setDefaultData(ElasticSearchParticipantDto esData); + + protected abstract boolean setDefaultData(); @Override public boolean generateDefaults(String studyGuid, String participantId) { @@ -37,6 +38,7 @@ public boolean generateDefaults(String studyGuid, String participantId) { return false; } instance = DDPInstance.getDDPInstance(studyGuid); - return setDefaultData(maybeParticipantESDataByParticipantId.get()); + elasticSearchParticipantDto = maybeParticipantESDataByParticipantId.get(); + return setDefaultData(); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/at/DefaultValues.java b/src/main/java/org/broadinstitute/dsm/model/defaultvalues/DefaultValues.java similarity index 96% rename from src/main/java/org/broadinstitute/dsm/model/at/DefaultValues.java rename to src/main/java/org/broadinstitute/dsm/model/defaultvalues/DefaultValues.java index 82ce02557..93e21469f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/at/DefaultValues.java +++ b/src/main/java/org/broadinstitute/dsm/model/defaultvalues/DefaultValues.java @@ -1,8 +1,7 @@ -package org.broadinstitute.dsm.model.at; +package org.broadinstitute.dsm.model.defaultvalues; import com.google.common.reflect.TypeToken; import com.google.gson.Gson; -import lombok.NonNull; import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.DDPInstance; import org.broadinstitute.dsm.db.dao.Dao; @@ -12,7 +11,6 @@ import org.broadinstitute.dsm.db.dto.bookmark.BookmarkDto; import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData; import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; -import org.broadinstitute.dsm.model.defaultvalues.BasicDefaultDataMaker; import org.broadinstitute.dsm.model.elastic.ESProfile; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.settings.field.FieldSettings; @@ -125,9 +123,12 @@ private boolean hasParticipantDataGenomicId(List participantDat }); } - private boolean isSelfOrDependentParticipant(List participantDataList) { - if (participantDataList.isEmpty()) return false; - return participantDataList.stream() + boolean isSelfOrDependentParticipant() { + + if (elasticSearchParticipantDto.getActivities().isEmpty()) return false; + + if (esDto.isEmpty()) return false; + return esDto.stream() .anyMatch(pData -> { Map data = GSON.fromJson(pData.getData().orElse(""), new TypeToken>() {}.getType()); return data.containsKey(REGISTRATION_TYPE) && @@ -234,7 +235,7 @@ private void setDataAccess(Dao dao) { } @Override - protected boolean setDefaultData(ElasticSearchParticipantDto esData) { + protected boolean setDefaultData() { isSelfOrDependentParticipant() return esData.getProfile() .map(esProfile -> { diff --git a/src/main/java/org/broadinstitute/dsm/model/defaultvalues/DefaultableMaker.java b/src/main/java/org/broadinstitute/dsm/model/defaultvalues/DefaultableMaker.java index d3ae0560d..9113286c1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/defaultvalues/DefaultableMaker.java +++ b/src/main/java/org/broadinstitute/dsm/model/defaultvalues/DefaultableMaker.java @@ -1,7 +1,6 @@ package org.broadinstitute.dsm.model.defaultvalues; import org.broadinstitute.dsm.model.Study; -import org.broadinstitute.dsm.model.defaultvalues.rgp.AutomaticProbandDataCreator; public class DefaultableMaker { @@ -11,7 +10,7 @@ public static Defaultable makeDefaultable(Study study) { case ATCP: break; case RGP: - defaultable = new AutomaticProbandDataCreator(); + defaultable = new RgpAutomaticProbandDataCreator(); break; default: break; diff --git a/src/main/java/org/broadinstitute/dsm/model/defaultvalues/rgp/AutomaticProbandDataCreator.java b/src/main/java/org/broadinstitute/dsm/model/defaultvalues/RgpAutomaticProbandDataCreator.java similarity index 98% rename from src/main/java/org/broadinstitute/dsm/model/defaultvalues/rgp/AutomaticProbandDataCreator.java rename to src/main/java/org/broadinstitute/dsm/model/defaultvalues/RgpAutomaticProbandDataCreator.java index 842c2283a..01623a894 100644 --- a/src/main/java/org/broadinstitute/dsm/model/defaultvalues/rgp/AutomaticProbandDataCreator.java +++ b/src/main/java/org/broadinstitute/dsm/model/defaultvalues/RgpAutomaticProbandDataCreator.java @@ -1,4 +1,4 @@ -package org.broadinstitute.dsm.model.defaultvalues.rgp; +package org.broadinstitute.dsm.model.defaultvalues; import java.util.List; import java.util.Map; @@ -23,7 +23,7 @@ import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; -public class AutomaticProbandDataCreator extends BasicDefaultDataMaker { +public class RgpAutomaticProbandDataCreator extends BasicDefaultDataMaker { public static final String RGP_FAMILY_ID = "rgp_family_id"; diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java index 72375f1f8..15eecbc00 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java @@ -16,7 +16,7 @@ import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchable; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; -import org.broadinstitute.dsm.model.at.DefaultValues; +import org.broadinstitute.dsm.model.defaultvalues.DefaultValues; import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.elasticsearch.index.query.BoolQueryBuilder; diff --git a/src/test/java/org/broadinstitute/dsm/model/defaultvalues/DefaultValuesTest.java b/src/test/java/org/broadinstitute/dsm/model/defaultvalues/DefaultValuesTest.java new file mode 100644 index 000000000..3b37a7aa6 --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/defaultvalues/DefaultValuesTest.java @@ -0,0 +1,36 @@ +package org.broadinstitute.dsm.model.defaultvalues; + +import org.broadinstitute.dsm.model.elastic.ESActivities; +import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; +import org.junit.Assert; +import org.junit.Test; + +import java.util.*; + +public class DefaultValuesTest { + + @Test + public void isSelfOrDependentParticipant() { + DefaultValues defaultValues = new DefaultValues(); + + ESActivities esActivities = new ESActivities(); + esActivities.setActivityCode("PREQUAL"); + esActivities.setQuestionsAnswers(Collections.singletonList(Map.of("answer", List.of("CHILD_DIAGNOSED")))); + + ESActivities esActivities2 = new ESActivities(); + esActivities2.setActivityCode("PREQUAL"); + esActivities2.setQuestionsAnswers(Collections.singletonList(Map.of("answer", List.of("DIAGNOSED")))); + + ElasticSearchParticipantDto participantDto = new ElasticSearchParticipantDto.Builder() + .withActivities(List.of(esActivities)) + .build(); + + defaultValues.elasticSearchParticipantDto = participantDto; + + Assert.assertTrue(defaultValues.isSelfOrDependentParticipant()); + + participantDto.setActivities(List.of(esActivities2)); + + Assert.assertTrue(defaultValues.isSelfOrDependentParticipant()); + } +} \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreatorTest.java b/src/test/java/org/broadinstitute/dsm/model/defaultvalues/RgpAutomaticProbandDataCreatorTest.java similarity index 79% rename from src/test/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreatorTest.java rename to src/test/java/org/broadinstitute/dsm/model/defaultvalues/RgpAutomaticProbandDataCreatorTest.java index 4ee536823..fed0141e8 100644 --- a/src/test/java/org/broadinstitute/dsm/model/rgp/AutomaticProbandDataCreatorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/defaultvalues/RgpAutomaticProbandDataCreatorTest.java @@ -1,17 +1,17 @@ -package org.broadinstitute.dsm.model.rgp; +package org.broadinstitute.dsm.model.defaultvalues; import static org.broadinstitute.dsm.TestHelper.setupDB; import java.util.Map; -import org.broadinstitute.dsm.model.defaultvalues.rgp.AutomaticProbandDataCreator; +import org.broadinstitute.dsm.model.defaultvalues.RgpAutomaticProbandDataCreator; import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; -public class AutomaticProbandDataCreatorTest { +public class RgpAutomaticProbandDataCreatorTest { @BeforeClass @@ -24,7 +24,7 @@ public void insertFamilyIdToEs() { int familyId = 999; String participantId = "S3POBS0P9X0MB41FT1JZ"; String esIndex = "participants_structured.rgp.rgp"; - new AutomaticProbandDataCreator().insertFamilyIdToDsmES(esIndex, participantId, + new RgpAutomaticProbandDataCreator().insertFamilyIdToDsmES(esIndex, participantId, familyId); int familyIdFromEs = 0; try { From 8d553b834abb3d39062944843caa0da1388f2444 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 31 Jan 2022 16:16:03 +0400 Subject: [PATCH 362/405] DDP-7443 | finish test of default value self/dependent checker --- .../model/defaultvalues/DefaultValues.java | 81 +++++++++++-------- .../RgpAutomaticProbandDataCreator.java | 13 ++- .../model/participant/ParticipantWrapper.java | 18 ++--- .../defaultvalues/DefaultValuesTest.java | 13 ++- 4 files changed, 70 insertions(+), 55 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/defaultvalues/DefaultValues.java b/src/main/java/org/broadinstitute/dsm/model/defaultvalues/DefaultValues.java index 93e21469f..5f985e053 100644 --- a/src/main/java/org/broadinstitute/dsm/model/defaultvalues/DefaultValues.java +++ b/src/main/java/org/broadinstitute/dsm/model/defaultvalues/DefaultValues.java @@ -11,6 +11,8 @@ import org.broadinstitute.dsm.db.dto.bookmark.BookmarkDto; import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData; import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; +import org.broadinstitute.dsm.model.ddp.DDPActivityConstants; +import org.broadinstitute.dsm.model.elastic.ESActivities; import org.broadinstitute.dsm.model.elastic.ESProfile; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.settings.field.FieldSettings; @@ -23,6 +25,7 @@ import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; public class DefaultValues extends BasicDefaultDataMaker { @@ -39,6 +42,11 @@ public class DefaultValues extends BasicDefaultDataMaker { private static final Gson GSON = new Gson(); public static final String AT_PARTICIPANT_EXIT = "AT_PARTICIPANT_EXIT"; public static final String AT_GENOMIC_ID = "at_genomic_id"; + public static final String ACTIVITY_CODE_PREQUAL = "PREQUAL"; + public static final String PREQUAL_SELF_DESCRIBE = "PREQUAL_SELF_DESCRIBE"; + public static final String QUESTION_ANSWER = "answer"; + public static final String SELF_DESCRIBE_CHILD_DIAGNOSED = "CHILD_DIAGNOSED"; + public static final String SELF_DESCRIBE_DIAGNOSED = "DIAGNOSED"; private Dao dataAccess; private Map> participantData; @@ -58,6 +66,7 @@ public class DefaultValues extends BasicDefaultDataMaker { // } public Map> addDefaultValues() { + if (participantESData == null) { logger.warn("Could not update default values, participant ES data is null"); return participantData; @@ -69,20 +78,20 @@ public Map> addDefaultValues() { String ddpParticipantId = entry.getKey(); List participantDataList = participantData.get(ddpParticipantId); if (participantDataList == null) continue; - - if (!hasParticipantDataGenomicId(participantDataList) && isSelfOrDependentParticipant(participantDataList)) { - ElasticSearchParticipantDto esParticipantData = entry.getValue(); - String hruid = esParticipantData.getProfile().map(ESProfile::getHruid).orElse(""); - addedNewParticipantData = getParticipantGenomicFieldData(participantDataList) - .map(pData -> insertGenomicIdIfNotExistsInData(ddpParticipantId, hruid, pData)) - .orElseGet(() -> insertGenomicIdForParticipant(ddpParticipantId, hruid)); - } - - if (!hasExitedStatusDefaultValue(participantDataList) && isSelfOrDependentParticipant(participantDataList)) { - addedNewParticipantData = getParticipantExitFieldData(participantDataList) - .map(pData -> insertExitStatusIfNotExistsInData(ddpParticipantId, pData)) - .orElseGet(() -> insertExitStatusForParticipant(ddpParticipantId)); - } +// +// if (!hasParticipantDataGenomicId(participantDataList) && isSelfOrDependentParticipant(participantDataList)) { +// ElasticSearchParticipantDto esParticipantData = entry.getValue(); +// String hruid = esParticipantData.getProfile().map(ESProfile::getHruid).orElse(""); +// addedNewParticipantData = getParticipantGenomicFieldData(participantDataList) +// .map(pData -> insertGenomicIdIfNotExistsInData(ddpParticipantId, hruid, pData)) +// .orElseGet(() -> insertGenomicIdForParticipant(ddpParticipantId, hruid)); +// } +// +// if (!hasExitedStatusDefaultValue(participantDataList) && isSelfOrDependentParticipant(participantDataList)) { +// addedNewParticipantData = getParticipantExitFieldData(participantDataList) +// .map(pData -> insertExitStatusIfNotExistsInData(ddpParticipantId, pData)) +// .orElseGet(() -> insertExitStatusForParticipant(ddpParticipantId)); +// } } if (addedNewParticipantData) { @@ -124,17 +133,17 @@ private boolean hasParticipantDataGenomicId(List participantDat } boolean isSelfOrDependentParticipant() { - if (elasticSearchParticipantDto.getActivities().isEmpty()) return false; - - if (esDto.isEmpty()) return false; - return esDto.stream() - .anyMatch(pData -> { - Map data = GSON.fromJson(pData.getData().orElse(""), new TypeToken>() {}.getType()); - return data.containsKey(REGISTRATION_TYPE) && - (REGISTRATION_TYPE_SELF.equalsIgnoreCase(data.get(REGISTRATION_TYPE)) - || REGISTRATION_TYPE_DEPENDENT.equalsIgnoreCase(data.get(REGISTRATION_TYPE))); - }); + return elasticSearchParticipantDto.getActivities().stream() + .anyMatch(this::isPrequalAndSelfOrDependent); + } + + private boolean isPrequalAndSelfOrDependent(ESActivities activity) { + return ACTIVITY_CODE_PREQUAL.equals(activity.getActivityCode()) && + (activity.getQuestionsAnswers().stream() + .filter(anwers -> PREQUAL_SELF_DESCRIBE.equals(anwers.get(DDPActivityConstants.DDP_ACTIVITY_STABLE_ID))) + .anyMatch(answers -> ((List)answers.get(QUESTION_ANSWER)).stream().anyMatch(answer -> SELF_DESCRIBE_CHILD_DIAGNOSED.equals(answer) || + SELF_DESCRIBE_DIAGNOSED.equals(answer)))); } private boolean hasExitedStatusDefaultValue(List participantDataList) { @@ -236,16 +245,18 @@ private void setDataAccess(Dao dao) { @Override protected boolean setDefaultData() { - isSelfOrDependentParticipant() - return esData.getProfile() - .map(esProfile -> { - return - insertGenomicIdForParticipant(ddpParticipantId, hruid); - insertExitStatusForParticipant(ddpParticipantId); - }) - .orElseGet(() -> { - logger.info("Participant does not have ES profile yet..."); - return false; - }); +// if (elasticSearchParticipantDto.get) +// return elasticSearchParticipantDto.getProfile() +// .map(esProfile -> { +// return +// insertGenomicIdForParticipant(ddpParticipantId, hruid); +// insertExitStatusForParticipant(ddpParticipantId); +// isSelfOrDependentParticipant() +// }) +// .orElseGet(() -> { +// logger.info("Participant does not have ES profile yet..."); +// return false; +// }); + return false; } } diff --git a/src/main/java/org/broadinstitute/dsm/model/defaultvalues/RgpAutomaticProbandDataCreator.java b/src/main/java/org/broadinstitute/dsm/model/defaultvalues/RgpAutomaticProbandDataCreator.java index 01623a894..a9a97d3cf 100644 --- a/src/main/java/org/broadinstitute/dsm/model/defaultvalues/RgpAutomaticProbandDataCreator.java +++ b/src/main/java/org/broadinstitute/dsm/model/defaultvalues/RgpAutomaticProbandDataCreator.java @@ -29,12 +29,12 @@ public class RgpAutomaticProbandDataCreator extends BasicDefaultDataMaker { public static final String RGP_FAMILY_ID = "rgp_family_id"; @Override - protected boolean setDefaultData(ElasticSearchParticipantDto esData) { + protected boolean setDefaultData() { List fieldSettingsDtosByOptionAndInstanceId = FieldSettingsDao.of().getOptionAndRadioFieldSettingsByInstanceId(Integer.parseInt(instance.getDdpInstanceId())); - return esData.getProfile() + return elasticSearchParticipantDto.getProfile() .map(esProfile -> { logger.info("Got ES profile of participant: " + esProfile.getGuid()); Map columnsWithDefaultOptions = @@ -46,7 +46,7 @@ protected boolean setDefaultData(ElasticSearchParticipantDto esData) { : esProfile.getGuid(); ParticipantData participantData = new ParticipantData(participantDataDao); Optional maybeFamilyIdOfBookmark = bookmarkDao.getBookmarkByInstance(RGP_FAMILY_ID); - Map probandDataMap = extractProbandDefaultDataFromParticipantProfile(esData, maybeFamilyIdOfBookmark); + Map probandDataMap = extractProbandDefaultDataFromParticipantProfile(maybeFamilyIdOfBookmark); participantData.setData( participantId, Integer.parseInt(instance.getDdpInstanceId()), @@ -83,11 +83,10 @@ protected boolean setDefaultData(ElasticSearchParticipantDto esData) { }); } - private Map extractProbandDefaultDataFromParticipantProfile(@NonNull ElasticSearchParticipantDto esData, - Optional maybeBookmark) { - List participantActivities = esData.getActivities(); + private Map extractProbandDefaultDataFromParticipantProfile(Optional maybeBookmark) { + List participantActivities = elasticSearchParticipantDto.getActivities(); String mobilePhone = getPhoneNumberFromActivities(participantActivities); - return esData.getProfile() + return elasticSearchParticipantDto.getProfile() .map(esProfile -> { logger.info("Starting extracting data from participant: " + esProfile.getGuid() + " ES profile"); String firstName = esProfile.getFirstName(); diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java index 15eecbc00..ae83c982a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java @@ -73,10 +73,10 @@ public ParticipantWrapperResult getFilteredList() { .orElseGet(() -> { fetchAndPrepareData(ddpInstance); //if study is AT - if ("atcp".equals(ddpInstance.getName())) { - DefaultValues defaultValues = new DefaultValues(participantData, esData.getEsParticipants(), ddpInstance, null); - participantData = defaultValues.addDefaultValues(); - } +// if ("atcp".equals(ddpInstance.getName())) { +// DefaultValues defaultValues = new DefaultValues(participantData, esData.getEsParticipants(), ddpInstance, null); +// participantData = defaultValues.addDefaultValues(); +// } sortBySelfElseById(participantData.values()); return new ParticipantWrapperResult(esData.getTotalCount(), collectData(ddpInstance)); }); @@ -98,11 +98,11 @@ else if (DBConstants.DDP_PARTICIPANT_DATA_ALIAS.equals(source)) { participantData = new ParticipantDataDao().getParticipantDataByInstanceIdAndFilterQuery(Integer.parseInt(ddpInstance.getDdpInstanceId()), filters.get(source)); //if study is AT TODO - if ("atcp".equals(ddpInstance.getName())) { - DefaultValues defaultValues = - new DefaultValues(participantData, esData.getEsParticipants(), ddpInstance, filters.get(source)); - participantData = defaultValues.addDefaultValues(); - } +// if ("atcp".equals(ddpInstance.getName())) { +// DefaultValues defaultValues = +// new DefaultValues(participantData, esData.getEsParticipants(), ddpInstance, filters.get(source)); +// participantData = defaultValues.addDefaultValues(); +// } } else if ("ES".equals(source)){ //source is not of any study-manager table so it must be ES diff --git a/src/test/java/org/broadinstitute/dsm/model/defaultvalues/DefaultValuesTest.java b/src/test/java/org/broadinstitute/dsm/model/defaultvalues/DefaultValuesTest.java index 3b37a7aa6..04ac6d546 100644 --- a/src/test/java/org/broadinstitute/dsm/model/defaultvalues/DefaultValuesTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/defaultvalues/DefaultValuesTest.java @@ -1,5 +1,6 @@ package org.broadinstitute.dsm.model.defaultvalues; +import org.broadinstitute.dsm.model.ddp.DDPActivityConstants; import org.broadinstitute.dsm.model.elastic.ESActivities; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.junit.Assert; @@ -14,12 +15,16 @@ public void isSelfOrDependentParticipant() { DefaultValues defaultValues = new DefaultValues(); ESActivities esActivities = new ESActivities(); - esActivities.setActivityCode("PREQUAL"); - esActivities.setQuestionsAnswers(Collections.singletonList(Map.of("answer", List.of("CHILD_DIAGNOSED")))); + esActivities.setActivityCode(DefaultValues.ACTIVITY_CODE_PREQUAL); + esActivities.setQuestionsAnswers(Collections.singletonList(Map.of( + DDPActivityConstants.DDP_ACTIVITY_STABLE_ID, DefaultValues.PREQUAL_SELF_DESCRIBE, + DefaultValues.QUESTION_ANSWER, List.of(DefaultValues.SELF_DESCRIBE_CHILD_DIAGNOSED)))); ESActivities esActivities2 = new ESActivities(); - esActivities2.setActivityCode("PREQUAL"); - esActivities2.setQuestionsAnswers(Collections.singletonList(Map.of("answer", List.of("DIAGNOSED")))); + esActivities2.setActivityCode(DefaultValues.ACTIVITY_CODE_PREQUAL); + esActivities2.setQuestionsAnswers(Collections.singletonList(Map.of( + DDPActivityConstants.DDP_ACTIVITY_STABLE_ID, DefaultValues.PREQUAL_SELF_DESCRIBE, + DefaultValues.QUESTION_ANSWER, List.of(DefaultValues.SELF_DESCRIBE_CHILD_DIAGNOSED)))); ElasticSearchParticipantDto participantDto = new ElasticSearchParticipantDto.Builder() .withActivities(List.of(esActivities)) From f86dc9e31ee0582904cc510c1067e3091832c149 Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 1 Feb 2022 10:10:10 +0400 Subject: [PATCH 363/405] DDP-7443 | replace field with "_id" for searching by participantId in ES script query --- .../elastic/export/painless/NestedUpsertPainlessFacade.java | 1 + .../dsm/model/participant/data/ParticipantData.java | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedUpsertPainlessFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedUpsertPainlessFacade.java index 0f23b9024..b1966854e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedUpsertPainlessFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedUpsertPainlessFacade.java @@ -23,6 +23,7 @@ protected ScriptBuilder buildScriptBuilder() { @Override protected QueryBuilder buildQueryBuilder() { MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder(fieldName, fieldValue); + if ("_id".equals(fieldName)) return matchQueryBuilder; return new NestedQueryBuilder(String.join(DBConstants.ALIAS_DELIMITER, ESObjectConstants.DSM, generator.getPropertyName(), fieldName), matchQueryBuilder, ScoreMode.Avg); } diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java b/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java index 6a4aff077..106106a4d 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java @@ -155,7 +155,7 @@ public long insertParticipantData(String userEmail) { .withDdpParticipantId(this.ddpParticipantId) .withDdpInstanceId(this.ddpInstanceId) .withFieldTypeId(this.fieldTypeId) - .withData(GSON.toJson(this.data)) + .withData(this.data) .withLastChanged(System.currentTimeMillis()) .withChangedBy(userEmail) .build(); @@ -202,7 +202,7 @@ public boolean updateParticipantData(int participantDataId, String changedByUser .withDdpParticipantId(this.ddpParticipantId) .withDdpInstanceId(this.ddpInstanceId) .withFieldTypeId(this.fieldTypeId) - .withData(GSON.toJson(this.data)) + .withData(this.data) .withLastChanged(System.currentTimeMillis()) .withChangedBy(changedByUser) .build(); From 48a11a534f349e9e7af455e13ab6a60e128dca94 Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 1 Feb 2022 10:40:38 +0400 Subject: [PATCH 364/405] DDP-7443 | continue implementing setDefaultData() method in DefaultValues.java --- .../model/defaultvalues/DefaultValues.java | 32 ++++++++++++------- .../painless/NestedUpsertPainlessFacade.java | 2 +- .../dsm/statics/ESObjectConstants.java | 1 + 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/defaultvalues/DefaultValues.java b/src/main/java/org/broadinstitute/dsm/model/defaultvalues/DefaultValues.java index 5f985e053..21cd72e0b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/defaultvalues/DefaultValues.java +++ b/src/main/java/org/broadinstitute/dsm/model/defaultvalues/DefaultValues.java @@ -245,18 +245,26 @@ private void setDataAccess(Dao dao) { @Override protected boolean setDefaultData() { -// if (elasticSearchParticipantDto.get) -// return elasticSearchParticipantDto.getProfile() -// .map(esProfile -> { -// return -// insertGenomicIdForParticipant(ddpParticipantId, hruid); -// insertExitStatusForParticipant(ddpParticipantId); -// isSelfOrDependentParticipant() -// }) -// .orElseGet(() -> { -// logger.info("Participant does not have ES profile yet..."); -// return false; -// }); + + if (isParticipantDataInES()) + return false; + + if (isSelfOrDependentParticipant()) + + return elasticSearchParticipantDto.getProfile() + .map(esProfile -> { + + insertGenomicIdForParticipant(ddpParticipantId, hruid); + insertExitStatusForParticipant(ddpParticipantId); + }) + .orElseGet(() -> { + return false; + }); return false; } + + private boolean isParticipantDataInES() { + logger.info("Participant does not have profile and activities in ES yet..."); + return elasticSearchParticipantDto.getProfile().isEmpty() && elasticSearchParticipantDto.getActivities().isEmpty(); + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedUpsertPainlessFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedUpsertPainlessFacade.java index b1966854e..132c1616e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedUpsertPainlessFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedUpsertPainlessFacade.java @@ -23,7 +23,7 @@ protected ScriptBuilder buildScriptBuilder() { @Override protected QueryBuilder buildQueryBuilder() { MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder(fieldName, fieldValue); - if ("_id".equals(fieldName)) return matchQueryBuilder; + if (ESObjectConstants.DOC_ID.equals(fieldName)) return matchQueryBuilder; return new NestedQueryBuilder(String.join(DBConstants.ALIAS_DELIMITER, ESObjectConstants.DSM, generator.getPropertyName(), fieldName), matchQueryBuilder, ScoreMode.Avg); } diff --git a/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java b/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java index b584f6bab..13173903c 100644 --- a/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java +++ b/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java @@ -76,4 +76,5 @@ public class ESObjectConstants { public static final String PARTICIPANT_RECORD = "participantRecord"; public static final String PARTICIPANT = "participant"; public static final String DYNAMIC_FIELDS = "dynamicFields"; + public static final String DOC_ID = "_id"; } From a1cd7b6c14fcb55759a76729ed4f356e8950428c Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 1 Feb 2022 11:11:04 +0400 Subject: [PATCH 365/405] DDP-7443 | minor changes on inserting methods, clean up dead code --- .../defaultvalues/BasicDefaultDataMaker.java | 3 +- .../model/defaultvalues/DefaultValues.java | 193 +++--------------- 2 files changed, 31 insertions(+), 165 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/defaultvalues/BasicDefaultDataMaker.java b/src/main/java/org/broadinstitute/dsm/model/defaultvalues/BasicDefaultDataMaker.java index da1caedf7..4dd040266 100644 --- a/src/main/java/org/broadinstitute/dsm/model/defaultvalues/BasicDefaultDataMaker.java +++ b/src/main/java/org/broadinstitute/dsm/model/defaultvalues/BasicDefaultDataMaker.java @@ -2,6 +2,7 @@ import java.util.Optional; +import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.DDPInstance; import org.broadinstitute.dsm.db.dao.bookmark.BookmarkDao; import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; @@ -30,7 +31,7 @@ public abstract class BasicDefaultDataMaker implements Defaultable { @Override public boolean generateDefaults(String studyGuid, String participantId) { String esParticipantIndex = ddpInstanceDao.getEsParticipantIndexByStudyGuid(studyGuid) - .orElse(""); + .orElse(StringUtils.EMPTY); Optional maybeParticipantESDataByParticipantId = ElasticSearchUtil.getParticipantESDataByParticipantId(esParticipantIndex, participantId); if (maybeParticipantESDataByParticipantId.isEmpty()) { diff --git a/src/main/java/org/broadinstitute/dsm/model/defaultvalues/DefaultValues.java b/src/main/java/org/broadinstitute/dsm/model/defaultvalues/DefaultValues.java index 21cd72e0b..ef3033846 100644 --- a/src/main/java/org/broadinstitute/dsm/model/defaultvalues/DefaultValues.java +++ b/src/main/java/org/broadinstitute/dsm/model/defaultvalues/DefaultValues.java @@ -1,45 +1,34 @@ package org.broadinstitute.dsm.model.defaultvalues; -import com.google.common.reflect.TypeToken; import com.google.gson.Gson; import org.apache.commons.lang3.StringUtils; -import org.broadinstitute.dsm.db.DDPInstance; import org.broadinstitute.dsm.db.dao.Dao; import org.broadinstitute.dsm.db.dao.bookmark.BookmarkDao; import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; import org.broadinstitute.dsm.db.dao.settings.FieldSettingsDao; import org.broadinstitute.dsm.db.dto.bookmark.BookmarkDto; -import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData; import org.broadinstitute.dsm.db.dto.settings.FieldSettingsDto; import org.broadinstitute.dsm.model.ddp.DDPActivityConstants; import org.broadinstitute.dsm.model.elastic.ESActivities; import org.broadinstitute.dsm.model.elastic.ESProfile; -import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.settings.field.FieldSettings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.function.Function; import java.util.stream.Collectors; -import java.util.stream.Stream; public class DefaultValues extends BasicDefaultDataMaker { private static final Logger logger = LoggerFactory.getLogger(DefaultValues.class); - private static final String FIELD_TYPE_ID = "AT_GROUP_GENOME_STUDY"; + private static final String GENOME_STUDY_FIELD_TYPE = "AT_GROUP_GENOME_STUDY"; private static final String GENOME_STUDY_CPT_ID = "GENOME_STUDY_CPT_ID"; private static final String PREFIX = "DDP_ATCP_"; - private static final String REGISTRATION_TYPE = "REGISTRATION_TYPE"; - public static final String REGISTRATION_TYPE_SELF = "self"; - public static final String REGISTRATION_TYPE_DEPENDENT = "dependent"; - public static final String EXITSTATUS = "EXITSTATUS"; + public static final String EXIT_STATUS = "EXITSTATUS"; - private static final Gson GSON = new Gson(); public static final String AT_PARTICIPANT_EXIT = "AT_PARTICIPANT_EXIT"; public static final String AT_GENOMIC_ID = "at_genomic_id"; public static final String ACTIVITY_CODE_PREQUAL = "PREQUAL"; @@ -49,88 +38,27 @@ public class DefaultValues extends BasicDefaultDataMaker { public static final String SELF_DESCRIBE_DIAGNOSED = "DIAGNOSED"; private Dao dataAccess; - private Map> participantData; - private List participantESData; - private DDPInstance instance; - private String queryAddition; - private ParticipantDataDao participantDataDao; - -// -// public DefaultValues(Map> participantData, -// List participantESData, @NonNull DDPInstance instance, String queryAddition) { -// this.participantData = participantData; -// this.participantESData = participantESData; -// this.instance = instance; -// this.queryAddition = queryAddition; -// this.participantDataDao = new ParticipantDataDao(); -// } - - public Map> addDefaultValues() { + @Override + protected boolean setDefaultData() { - if (participantESData == null) { - logger.warn("Could not update default values, participant ES data is null"); - return participantData; - } - boolean addedNewParticipantData = false; - Map participantEsDataByParticipantId = - participantESData.stream().collect(Collectors.toMap(ElasticSearchParticipantDto::getParticipantId, Function.identity())); - for (Map.Entry entry: participantEsDataByParticipantId.entrySet()) { - String ddpParticipantId = entry.getKey(); - List participantDataList = participantData.get(ddpParticipantId); - if (participantDataList == null) continue; -// -// if (!hasParticipantDataGenomicId(participantDataList) && isSelfOrDependentParticipant(participantDataList)) { -// ElasticSearchParticipantDto esParticipantData = entry.getValue(); -// String hruid = esParticipantData.getProfile().map(ESProfile::getHruid).orElse(""); -// addedNewParticipantData = getParticipantGenomicFieldData(participantDataList) -// .map(pData -> insertGenomicIdIfNotExistsInData(ddpParticipantId, hruid, pData)) -// .orElseGet(() -> insertGenomicIdForParticipant(ddpParticipantId, hruid)); -// } -// -// if (!hasExitedStatusDefaultValue(participantDataList) && isSelfOrDependentParticipant(participantDataList)) { -// addedNewParticipantData = getParticipantExitFieldData(participantDataList) -// .map(pData -> insertExitStatusIfNotExistsInData(ddpParticipantId, pData)) -// .orElseGet(() -> insertExitStatusForParticipant(ddpParticipantId)); -// } + if (isParticipantDataInES()) + return false; + if (isSelfOrDependentParticipant()) { + return insertGenomicIdForParticipant() + && insertExitStatusForParticipant(); + } else { + //in case if 3rd registration option is chosen in prequalifier of ATCP + //which is just stay inform registration + return true; } - if (addedNewParticipantData) { - //participant data was added, getting new list of data - if (StringUtils.isNotBlank(queryAddition)) { - List participantDataByInstanceIdAndQueryAddition = - participantDataDao.getParticipantDataByInstanceIdAndQueryAddition(Integer.parseInt(instance.getDdpInstanceId()), - queryAddition); - participantData = getParticipantDataWithParticipantId(participantDataByInstanceIdAndQueryAddition); - } - else { - List participantDataByInstanceId = - participantDataDao.getParticipantDataByInstanceId(Integer.parseInt(instance.getDdpInstanceId())); - participantData = getParticipantDataWithParticipantId(participantDataByInstanceId); - } - } - return participantData; } - private Map> getParticipantDataWithParticipantId(List participantDataByInstanceId) { - return participantDataByInstanceId.stream() - .collect(Collectors.toMap( - pDataDto -> pDataDto.getDdpParticipantId().orElse(""), - pDataDto -> new ArrayList<>(List.of(pDataDto)), - (prev, curr) -> { - prev.addAll(curr); - return prev; - } - )); + private boolean isParticipantDataInES() { + logger.info("Participant does not have profile and activities in ES yet..."); + return elasticSearchParticipantDto.getProfile().isEmpty() && elasticSearchParticipantDto.getActivities().isEmpty(); } - private boolean hasParticipantDataGenomicId(List participantDataList) { - if (participantDataList.isEmpty()) return false; - return participantDataList.stream() - .anyMatch(participantData -> { - Map data = GSON.fromJson(participantData.getData().orElse(""), new TypeToken>() {}.getType()); - return data.containsKey(GENOME_STUDY_CPT_ID) && StringUtils.isNotBlank(data.get(GENOME_STUDY_CPT_ID)); - }); - } boolean isSelfOrDependentParticipant() { if (elasticSearchParticipantDto.getActivities().isEmpty()) return false; @@ -146,30 +74,6 @@ private boolean isPrequalAndSelfOrDependent(ESActivities activity) { SELF_DESCRIBE_DIAGNOSED.equals(answer)))); } - private boolean hasExitedStatusDefaultValue(List participantDataList) { - if (participantDataList.isEmpty()) return false; - return participantDataList.stream() - .anyMatch(participantData -> { - Map data = GSON.fromJson(participantData.getData().orElse(""), new TypeToken>() {}.getType()); - return AT_PARTICIPANT_EXIT.equals(participantData.getFieldTypeId().orElse("")) && (data.containsKey(EXITSTATUS) && StringUtils.isNotBlank(data.get(EXITSTATUS))); - }); - } - - private Optional getParticipantGenomicFieldData(List participantDataList) { - if (participantDataList.isEmpty()) return Optional.empty(); - return participantDataList.stream() - .filter(participantData -> FIELD_TYPE_ID.equals(participantData.getFieldTypeId().orElse(""))) - .findFirst(); - } - - private boolean insertGenomicIdIfNotExistsInData(String ddpParticipantId, String hruid, - ParticipantData pData) { - Map dataMap = GSON.fromJson(pData.getData().orElse(""), new TypeToken>() {}.getType()); - if (dataMap.containsKey(GENOME_STUDY_CPT_ID)) return false; - dataMap.put(GENOME_STUDY_CPT_ID, PREFIX.concat(getGenomicIdValue(hruid))); - return updateParticipantData(ddpParticipantId, pData, dataMap); - } - private String getGenomicIdValue(String hruid) { this.setDataAccess(new BookmarkDao()); BookmarkDao dataAccess = (BookmarkDao) this.dataAccess; @@ -182,35 +86,18 @@ private String getGenomicIdValue(String hruid) { .orElse(hruid); } - private boolean insertGenomicIdForParticipant(String ddpParticipantId, String hruid) { - if (StringUtils.isBlank(hruid)) return false; - return insertParticipantData(Map.of(GENOME_STUDY_CPT_ID, PREFIX.concat(getGenomicIdValue(hruid))), ddpParticipantId, FIELD_TYPE_ID); - } - - private Optional getParticipantExitFieldData(List participantDataList) { - if (participantDataList.isEmpty()) return Optional.empty(); - return participantDataList.stream() - .filter(participantData -> AT_PARTICIPANT_EXIT.equals(participantData.getFieldTypeId().orElse(""))) - .findFirst(); - } - - private boolean insertExitStatusIfNotExistsInData(String ddpParticipantId, ParticipantData pData) { - Map dataMap = GSON.fromJson(pData.getData().orElse(""), new TypeToken>() {}.getType()); - if (dataMap.containsKey(EXITSTATUS)) return false; - dataMap.put(EXITSTATUS, getDefaultExitStatus()); - return updateParticipantData(ddpParticipantId, pData, dataMap); + private boolean insertGenomicIdForParticipant() { + ESProfile esProfile = elasticSearchParticipantDto.getProfile().orElseThrow(); + String ddpParticipantId = esProfile.getGuid(); + String hruid = esProfile.getHruid(); + return insertParticipantData(Map.of(GENOME_STUDY_CPT_ID, PREFIX.concat(getGenomicIdValue(hruid))), ddpParticipantId, + GENOME_STUDY_FIELD_TYPE); } - private boolean insertExitStatusForParticipant(String ddpParticipantId) { + private boolean insertExitStatusForParticipant() { + String ddpParticipantId = elasticSearchParticipantDto.getProfile().orElseThrow().getGuid(); String datstatExitReasonDefaultOption = getDefaultExitStatus(); - return insertParticipantData(Map.of(EXITSTATUS, datstatExitReasonDefaultOption), ddpParticipantId, AT_PARTICIPANT_EXIT); - } - - private boolean updateParticipantData(String ddpParticipantId, ParticipantData pData, Map dataMap) { - this.setDataAccess(new ParticipantDataDao()); - org.broadinstitute.dsm.model.participant.data.ParticipantData participantData = new org.broadinstitute.dsm.model.participant.data.ParticipantData(dataAccess); - participantData.setData(ddpParticipantId, Integer.parseInt(instance.getDdpInstanceId()), pData.getFieldTypeId().orElse(""), dataMap); - return participantData.updateParticipantData(pData.getParticipantDataId(), "SYSTEM"); + return insertParticipantData(Map.of(EXIT_STATUS, datstatExitReasonDefaultOption), ddpParticipantId, AT_PARTICIPANT_EXIT); } private boolean insertParticipantData(Map data, String ddpParticipantId, String fieldTypeId) { @@ -220,7 +107,8 @@ private boolean insertParticipantData(Map data, String ddpPartic fieldTypeId, data); try { participantData.insertParticipantData("SYSTEM"); - logger.info("values: " + data.keySet().stream().collect(Collectors.joining(", ", "[", "]")) + " were created for participant with id: " + ddpParticipantId + " at " + FIELD_TYPE_ID); + logger.info("values: " + data.keySet().stream().collect(Collectors.joining(", ", "[", "]")) + " were created for participant with id: " + ddpParticipantId + " at " + + GENOME_STUDY_FIELD_TYPE); return true; } catch (RuntimeException re) { return false; @@ -230,41 +118,18 @@ private boolean insertParticipantData(Map data, String ddpPartic private String getDefaultExitStatus() { this.setDataAccess(FieldSettingsDao.of()); Optional fieldSettingByColumnNameAndInstanceId = ((FieldSettingsDao) dataAccess) - .getFieldSettingByColumnNameAndInstanceId(Integer.parseInt(instance.getDdpInstanceId()), EXITSTATUS); + .getFieldSettingByColumnNameAndInstanceId(Integer.parseInt(instance.getDdpInstanceId()), EXIT_STATUS); return fieldSettingByColumnNameAndInstanceId. map(fieldSettingsDto -> { FieldSettings fieldSettings = new FieldSettings(); return fieldSettings.getDefaultValue(fieldSettingsDto.getPossibleValues()); }) - .orElse(""); + .orElse(StringUtils.EMPTY); } private void setDataAccess(Dao dao) { this.dataAccess = dao; } - @Override - protected boolean setDefaultData() { - - if (isParticipantDataInES()) - return false; - - if (isSelfOrDependentParticipant()) - - return elasticSearchParticipantDto.getProfile() - .map(esProfile -> { - - insertGenomicIdForParticipant(ddpParticipantId, hruid); - insertExitStatusForParticipant(ddpParticipantId); - }) - .orElseGet(() -> { - return false; - }); - return false; - } - private boolean isParticipantDataInES() { - logger.info("Participant does not have profile and activities in ES yet..."); - return elasticSearchParticipantDto.getProfile().isEmpty() && elasticSearchParticipantDto.getActivities().isEmpty(); - } } From 01b210a4ec1067ab266750b2a5444b62f270c405 Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 1 Feb 2022 15:05:59 +0400 Subject: [PATCH 366/405] DDP-7443 | remove unused tests, add ATCP case in DefaultValues, refactor code further --- ...efaultValues.java => ATDefaultValues.java} | 47 +++++++++---------- .../model/defaultvalues/DefaultableMaker.java | 1 + .../RgpAutomaticProbandDataCreator.java | 8 ---- .../model/participant/ParticipantWrapper.java | 1 - .../participant/data/ParticipantData.java | 1 + ...luesTest.java => ATDefaultValuesTest.java} | 16 +++---- .../RgpAutomaticProbandDataCreatorTest.java | 40 ---------------- 7 files changed, 33 insertions(+), 81 deletions(-) rename src/main/java/org/broadinstitute/dsm/model/defaultvalues/{DefaultValues.java => ATDefaultValues.java} (95%) rename src/test/java/org/broadinstitute/dsm/model/defaultvalues/{DefaultValuesTest.java => ATDefaultValuesTest.java} (62%) delete mode 100644 src/test/java/org/broadinstitute/dsm/model/defaultvalues/RgpAutomaticProbandDataCreatorTest.java diff --git a/src/main/java/org/broadinstitute/dsm/model/defaultvalues/DefaultValues.java b/src/main/java/org/broadinstitute/dsm/model/defaultvalues/ATDefaultValues.java similarity index 95% rename from src/main/java/org/broadinstitute/dsm/model/defaultvalues/DefaultValues.java rename to src/main/java/org/broadinstitute/dsm/model/defaultvalues/ATDefaultValues.java index ef3033846..3c685aa23 100644 --- a/src/main/java/org/broadinstitute/dsm/model/defaultvalues/DefaultValues.java +++ b/src/main/java/org/broadinstitute/dsm/model/defaultvalues/ATDefaultValues.java @@ -1,6 +1,5 @@ package org.broadinstitute.dsm.model.defaultvalues; -import com.google.gson.Gson; import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.dao.Dao; import org.broadinstitute.dsm.db.dao.bookmark.BookmarkDao; @@ -20,9 +19,9 @@ import java.util.Optional; import java.util.stream.Collectors; -public class DefaultValues extends BasicDefaultDataMaker { +public class ATDefaultValues extends BasicDefaultDataMaker { - private static final Logger logger = LoggerFactory.getLogger(DefaultValues.class); + private static final Logger logger = LoggerFactory.getLogger(ATDefaultValues.class); private static final String GENOME_STUDY_FIELD_TYPE = "AT_GROUP_GENOME_STUDY"; private static final String GENOME_STUDY_CPT_ID = "GENOME_STUDY_CPT_ID"; @@ -70,10 +69,18 @@ private boolean isPrequalAndSelfOrDependent(ESActivities activity) { return ACTIVITY_CODE_PREQUAL.equals(activity.getActivityCode()) && (activity.getQuestionsAnswers().stream() .filter(anwers -> PREQUAL_SELF_DESCRIBE.equals(anwers.get(DDPActivityConstants.DDP_ACTIVITY_STABLE_ID))) - .anyMatch(answers -> ((List)answers.get(QUESTION_ANSWER)).stream().anyMatch(answer -> SELF_DESCRIBE_CHILD_DIAGNOSED.equals(answer) || + .anyMatch(answers -> ((List) answers.get(QUESTION_ANSWER)).stream().anyMatch(answer -> SELF_DESCRIBE_CHILD_DIAGNOSED.equals(answer) || SELF_DESCRIBE_DIAGNOSED.equals(answer)))); } + private boolean insertGenomicIdForParticipant() { + ESProfile esProfile = elasticSearchParticipantDto.getProfile().orElseThrow(); + String ddpParticipantId = esProfile.getGuid(); + String hruid = esProfile.getHruid(); + return insertParticipantData(Map.of(GENOME_STUDY_CPT_ID, PREFIX.concat(getGenomicIdValue(hruid))), ddpParticipantId, + GENOME_STUDY_FIELD_TYPE); + } + private String getGenomicIdValue(String hruid) { this.setDataAccess(new BookmarkDao()); BookmarkDao dataAccess = (BookmarkDao) this.dataAccess; @@ -86,20 +93,24 @@ private String getGenomicIdValue(String hruid) { .orElse(hruid); } - private boolean insertGenomicIdForParticipant() { - ESProfile esProfile = elasticSearchParticipantDto.getProfile().orElseThrow(); - String ddpParticipantId = esProfile.getGuid(); - String hruid = esProfile.getHruid(); - return insertParticipantData(Map.of(GENOME_STUDY_CPT_ID, PREFIX.concat(getGenomicIdValue(hruid))), ddpParticipantId, - GENOME_STUDY_FIELD_TYPE); - } - private boolean insertExitStatusForParticipant() { String ddpParticipantId = elasticSearchParticipantDto.getProfile().orElseThrow().getGuid(); String datstatExitReasonDefaultOption = getDefaultExitStatus(); return insertParticipantData(Map.of(EXIT_STATUS, datstatExitReasonDefaultOption), ddpParticipantId, AT_PARTICIPANT_EXIT); } + private String getDefaultExitStatus() { + this.setDataAccess(FieldSettingsDao.of()); + Optional fieldSettingByColumnNameAndInstanceId = ((FieldSettingsDao) dataAccess) + .getFieldSettingByColumnNameAndInstanceId(Integer.parseInt(instance.getDdpInstanceId()), EXIT_STATUS); + return fieldSettingByColumnNameAndInstanceId. + map(fieldSettingsDto -> { + FieldSettings fieldSettings = new FieldSettings(); + return fieldSettings.getDefaultValue(fieldSettingsDto.getPossibleValues()); + }) + .orElse(StringUtils.EMPTY); + } + private boolean insertParticipantData(Map data, String ddpParticipantId, String fieldTypeId) { this.setDataAccess(new ParticipantDataDao()); org.broadinstitute.dsm.model.participant.data.ParticipantData participantData = new org.broadinstitute.dsm.model.participant.data.ParticipantData(dataAccess); @@ -115,18 +126,6 @@ private boolean insertParticipantData(Map data, String ddpPartic } } - private String getDefaultExitStatus() { - this.setDataAccess(FieldSettingsDao.of()); - Optional fieldSettingByColumnNameAndInstanceId = ((FieldSettingsDao) dataAccess) - .getFieldSettingByColumnNameAndInstanceId(Integer.parseInt(instance.getDdpInstanceId()), EXIT_STATUS); - return fieldSettingByColumnNameAndInstanceId. - map(fieldSettingsDto -> { - FieldSettings fieldSettings = new FieldSettings(); - return fieldSettings.getDefaultValue(fieldSettingsDto.getPossibleValues()); - }) - .orElse(StringUtils.EMPTY); - } - private void setDataAccess(Dao dao) { this.dataAccess = dao; } diff --git a/src/main/java/org/broadinstitute/dsm/model/defaultvalues/DefaultableMaker.java b/src/main/java/org/broadinstitute/dsm/model/defaultvalues/DefaultableMaker.java index 9113286c1..041f43718 100644 --- a/src/main/java/org/broadinstitute/dsm/model/defaultvalues/DefaultableMaker.java +++ b/src/main/java/org/broadinstitute/dsm/model/defaultvalues/DefaultableMaker.java @@ -8,6 +8,7 @@ public static Defaultable makeDefaultable(Study study) { Defaultable defaultable = (studyGuid, participantId) -> true; switch (study) { case ATCP: + defaultable = new ATDefaultValues(); break; case RGP: defaultable = new RgpAutomaticProbandDataCreator(); diff --git a/src/main/java/org/broadinstitute/dsm/model/defaultvalues/RgpAutomaticProbandDataCreator.java b/src/main/java/org/broadinstitute/dsm/model/defaultvalues/RgpAutomaticProbandDataCreator.java index a9a97d3cf..2ebcdbb31 100644 --- a/src/main/java/org/broadinstitute/dsm/model/defaultvalues/RgpAutomaticProbandDataCreator.java +++ b/src/main/java/org/broadinstitute/dsm/model/defaultvalues/RgpAutomaticProbandDataCreator.java @@ -56,14 +56,6 @@ protected boolean setDefaultData() { participantData.addDefaultOptionsValueToData(columnsWithDefaultOptions); participantData.insertParticipantData("SYSTEM"); - DDPInstanceDto ddpInstanceDto = new DDPInstanceDto.Builder() - .withInstanceName(instance.getName()) - .withEsParticipantIndex(instance.getParticipantIndexES()) - .build(); - - UpsertPainlessFacade.of(DBConstants.DDP_PARTICIPANT_DATA_ALIAS, participantData, ddpInstanceDto, "participantDataId", "_id", esProfile.getGuid()) - .export(); - columnsWithDefaultOptionsFilteredByElasticExportWorkflow.forEach((col, val) -> ElasticSearchUtil.writeWorkflow(WorkflowForES.createInstanceWithStudySpecificData(instance, participantId, col, val, new WorkflowForES.StudySpecificData(probandDataMap.get(FamilyMemberConstants.COLLABORATOR_PARTICIPANT_ID), diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java index ae83c982a..ed20984bf 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java @@ -16,7 +16,6 @@ import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchable; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; -import org.broadinstitute.dsm.model.defaultvalues.DefaultValues; import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.elasticsearch.index.query.BoolQueryBuilder; diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java b/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java index 106106a4d..07f2f6c64 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java @@ -166,6 +166,7 @@ public long insertParticipantData(String userEmail) { if (createdDataKey < 1) { throw new RuntimeException("Could not insert participant data for : " + this.ddpParticipantId); } + participantData.setParticipantDataId(createdDataKey); logger.info("Successfully inserted data for participant: " + this.ddpParticipantId); DDPInstanceDto ddpInstanceDto = new DDPInstanceDao().getDDPInstanceByInstanceId(ddpInstanceId).orElseThrow(); diff --git a/src/test/java/org/broadinstitute/dsm/model/defaultvalues/DefaultValuesTest.java b/src/test/java/org/broadinstitute/dsm/model/defaultvalues/ATDefaultValuesTest.java similarity index 62% rename from src/test/java/org/broadinstitute/dsm/model/defaultvalues/DefaultValuesTest.java rename to src/test/java/org/broadinstitute/dsm/model/defaultvalues/ATDefaultValuesTest.java index 04ac6d546..6c37985b5 100644 --- a/src/test/java/org/broadinstitute/dsm/model/defaultvalues/DefaultValuesTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/defaultvalues/ATDefaultValuesTest.java @@ -8,23 +8,23 @@ import java.util.*; -public class DefaultValuesTest { +public class ATDefaultValuesTest { @Test public void isSelfOrDependentParticipant() { - DefaultValues defaultValues = new DefaultValues(); + ATDefaultValues defaultValues = new ATDefaultValues(); ESActivities esActivities = new ESActivities(); - esActivities.setActivityCode(DefaultValues.ACTIVITY_CODE_PREQUAL); + esActivities.setActivityCode(ATDefaultValues.ACTIVITY_CODE_PREQUAL); esActivities.setQuestionsAnswers(Collections.singletonList(Map.of( - DDPActivityConstants.DDP_ACTIVITY_STABLE_ID, DefaultValues.PREQUAL_SELF_DESCRIBE, - DefaultValues.QUESTION_ANSWER, List.of(DefaultValues.SELF_DESCRIBE_CHILD_DIAGNOSED)))); + DDPActivityConstants.DDP_ACTIVITY_STABLE_ID, ATDefaultValues.PREQUAL_SELF_DESCRIBE, + ATDefaultValues.QUESTION_ANSWER, List.of(ATDefaultValues.SELF_DESCRIBE_CHILD_DIAGNOSED)))); ESActivities esActivities2 = new ESActivities(); - esActivities2.setActivityCode(DefaultValues.ACTIVITY_CODE_PREQUAL); + esActivities2.setActivityCode(ATDefaultValues.ACTIVITY_CODE_PREQUAL); esActivities2.setQuestionsAnswers(Collections.singletonList(Map.of( - DDPActivityConstants.DDP_ACTIVITY_STABLE_ID, DefaultValues.PREQUAL_SELF_DESCRIBE, - DefaultValues.QUESTION_ANSWER, List.of(DefaultValues.SELF_DESCRIBE_CHILD_DIAGNOSED)))); + DDPActivityConstants.DDP_ACTIVITY_STABLE_ID, ATDefaultValues.PREQUAL_SELF_DESCRIBE, + ATDefaultValues.QUESTION_ANSWER, List.of(ATDefaultValues.SELF_DESCRIBE_CHILD_DIAGNOSED)))); ElasticSearchParticipantDto participantDto = new ElasticSearchParticipantDto.Builder() .withActivities(List.of(esActivities)) diff --git a/src/test/java/org/broadinstitute/dsm/model/defaultvalues/RgpAutomaticProbandDataCreatorTest.java b/src/test/java/org/broadinstitute/dsm/model/defaultvalues/RgpAutomaticProbandDataCreatorTest.java deleted file mode 100644 index fed0141e8..000000000 --- a/src/test/java/org/broadinstitute/dsm/model/defaultvalues/RgpAutomaticProbandDataCreatorTest.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.broadinstitute.dsm.model.defaultvalues; - -import static org.broadinstitute.dsm.TestHelper.setupDB; - -import java.util.Map; - -import org.broadinstitute.dsm.model.defaultvalues.RgpAutomaticProbandDataCreator; -import org.broadinstitute.dsm.statics.ESObjectConstants; -import org.broadinstitute.dsm.util.ElasticSearchUtil; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; - -public class RgpAutomaticProbandDataCreatorTest { - - - @BeforeClass - public static void before() throws Exception { - setupDB(); - } - - @Test - public void insertFamilyIdToEs() { - int familyId = 999; - String participantId = "S3POBS0P9X0MB41FT1JZ"; - String esIndex = "participants_structured.rgp.rgp"; - new RgpAutomaticProbandDataCreator().insertFamilyIdToDsmES(esIndex, participantId, - familyId); - int familyIdFromEs = 0; - try { - Map esObjectsMap = ElasticSearchUtil.getObjectsMap(esIndex, participantId, ESObjectConstants.DSM); - Map dsmObjectMap = (Map) esObjectsMap.get(ESObjectConstants.DSM); - familyIdFromEs = (Integer) dsmObjectMap.get(ESObjectConstants.FAMILY_ID); - } catch (Exception e) { - e.printStackTrace(); - } - Assert.assertEquals(familyId, familyIdFromEs); - } - -} \ No newline at end of file From f18617e38991e1966dbc6909af21bed7a26d094b Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 1 Feb 2022 16:37:01 +0400 Subject: [PATCH 367/405] DDP-7443 | minor improvements --- .../dsm/model/participant/data/ParticipantData.java | 11 +++-------- .../dsm/util/proxy/jackson/ObjectMapperSingleton.java | 2 ++ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java b/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java index 07f2f6c64..ffa52cdfa 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java @@ -106,19 +106,14 @@ public static ParticipantData parseDto(@NonNull org.broadinstitute.dsm.db.dto.dd participantData.getDdpParticipantId().orElse(StringUtils.EMPTY), participantData.getDdpInstanceId(), participantData.getFieldTypeId().orElse(StringUtils.EMPTY), - GSON.fromJson(participantData.getData().orElse(StringUtils.EMPTY), new TypeToken>() {}.getType()) + ObjectMapperSingleton.readValue(participantData.getData().orElse(StringUtils.EMPTY), new TypeReference>() {}) ); } public static List parseDtoList(@NonNull List participantDataList) { List participantData = new ArrayList<>(); - participantDataList.forEach(dto -> participantData.add(new ParticipantData( - dto.getParticipantDataId(), - dto.getDdpParticipantId().orElse(StringUtils.EMPTY), - dto.getDdpInstanceId(), - dto.getFieldTypeId().orElse(StringUtils.EMPTY), - GSON.fromJson(dto.getData().orElse(StringUtils.EMPTY), new TypeToken>() {}.getType()) - ))); + participantDataList.forEach(dto -> participantData.add(parseDto(dto))); return participantData; } diff --git a/src/main/java/org/broadinstitute/dsm/util/proxy/jackson/ObjectMapperSingleton.java b/src/main/java/org/broadinstitute/dsm/util/proxy/jackson/ObjectMapperSingleton.java index bcd2476c4..b9789a551 100644 --- a/src/main/java/org/broadinstitute/dsm/util/proxy/jackson/ObjectMapperSingleton.java +++ b/src/main/java/org/broadinstitute/dsm/util/proxy/jackson/ObjectMapperSingleton.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.lang3.StringUtils; public class ObjectMapperSingleton { @@ -14,6 +15,7 @@ public static ObjectMapper instance() { } public static T readValue(String content, TypeReference typeReference) { + content = StringUtils.isBlank(content) ? "{}" : content; try { return Helper.objectMapperInstance.readValue(content, typeReference); } catch (com.fasterxml.jackson.core.JsonParseException e) { From 5dbfd763aac57c831e31c58545f0896b72cfc27c Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 1 Feb 2022 17:07:46 +0400 Subject: [PATCH 368/405] DDP-7443 | add LocalParticipantData helper for serializing object properly --- .../participant/data/ParticipantData.java | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java b/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java index ffa52cdfa..ae3825ef2 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java @@ -53,18 +53,18 @@ public class ParticipantData { public static final Gson GSON = new Gson(); @ColumnName(DBConstants.PARTICIPANT_DATA_ID) - private long participantDataId; + long participantDataId; @ColumnName(DBConstants.DDP_PARTICIPANT_ID) - private String ddpParticipantId; + String ddpParticipantId; - private int ddpInstanceId; + int ddpInstanceId; @ColumnName(DBConstants.FIELD_TYPE_ID) - private String fieldTypeId; + String fieldTypeId; @ColumnName (DBConstants.DATA) - private String data; + String data; @JsonProperty("dynamicFields") public Map getDynamicFields() { @@ -83,7 +83,7 @@ public Map getData() { return ObjectMapperSingleton.readValue(data, new TypeReference>() {}); } - private Dao dataAccess; + private transient Dao dataAccess; public ParticipantData() {} @@ -101,7 +101,21 @@ public ParticipantData(long participantDataId, String ddpParticipantId, int ddpI } public static ParticipantData parseDto(@NonNull org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData participantData) { - return new ParticipantData( + + class LocalParticipantData extends ParticipantData { + + @JsonProperty("data") + private Map data; + + public LocalParticipantData(long participantDataId, String ddpParticipantId, int ddpInstanceId, String fieldTypeId, Map data) { + this.participantDataId = participantDataId; + this.ddpParticipantId = ddpParticipantId; + this.ddpInstanceId = ddpInstanceId; + this.fieldTypeId = fieldTypeId; + this.data = data; + } + } + return new LocalParticipantData( participantData.getParticipantDataId(), participantData.getDdpParticipantId().orElse(StringUtils.EMPTY), participantData.getDdpInstanceId(), From aeb0467e15285cc02251399f4a99d1f80616d653 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 1 Feb 2022 17:37:18 +0400 Subject: [PATCH 369/405] DDP-7443 | continue process of clean up --- .../model/familymember/AddFamilyMember.java | 4 - .../participant/data/ParticipantData.java | 94 +++---------------- 2 files changed, 13 insertions(+), 85 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/familymember/AddFamilyMember.java b/src/main/java/org/broadinstitute/dsm/model/familymember/AddFamilyMember.java index 5974f1449..dacb2b803 100644 --- a/src/main/java/org/broadinstitute/dsm/model/familymember/AddFamilyMember.java +++ b/src/main/java/org/broadinstitute/dsm/model/familymember/AddFamilyMember.java @@ -99,10 +99,6 @@ public void exportDataToEs() { } else { exportDefaultWorkflowsForFamilyMemberToES(); } - DDPInstanceDto ddpInstanceDto = ddpInstanceDao.getDDPInstanceByGuid(studyGuid).orElseThrow(); - String participantGuid = Exportable.getParticipantGuid(ddpParticipantId, ddpInstanceDto.getEsParticipantIndex()); - UpsertPainlessFacade.of(DBConstants.DDP_PARTICIPANT_DATA_ALIAS, participantData, ddpInstanceDto, "participantDataId", "_id", participantGuid) - .export(); } protected void exportProbandDataForFamilyMemberToEs() { diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java b/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java index ae3825ef2..190c3d144 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/data/ParticipantData.java @@ -38,55 +38,32 @@ import org.slf4j.LoggerFactory; @Data -@TableName( - name = DBConstants.DDP_PARTICIPANT_DATA, - alias = DBConstants.DDP_PARTICIPANT_DATA_ALIAS, - primaryKey = DBConstants.PARTICIPANT_DATA_ID, - columnPrefix = "") -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown = true) public class ParticipantData { private static final Logger logger = LoggerFactory.getLogger(ParticipantData.class); public static final String FIELD_TYPE_PARTICIPANTS = "_PARTICIPANTS"; - public static final Gson GSON = new Gson(); - @ColumnName(DBConstants.PARTICIPANT_DATA_ID) - long participantDataId; + private long participantDataId; - @ColumnName(DBConstants.DDP_PARTICIPANT_ID) - String ddpParticipantId; + private String ddpParticipantId; - int ddpInstanceId; + private int ddpInstanceId; - @ColumnName(DBConstants.FIELD_TYPE_ID) - String fieldTypeId; + private String fieldTypeId; - @ColumnName (DBConstants.DATA) - String data; - - @JsonProperty("dynamicFields") - public Map getDynamicFields() { - try { - return ObjectMapperSingleton.readValue(data, new TypeReference>() {}); - } catch (Exception e) { - return Map.of(); - } - } + private Map data; public void setData(Map data) { - this.data = ObjectMapperSingleton.writeValueAsString(data); + this.data = data; } public Map getData() { - return ObjectMapperSingleton.readValue(data, new TypeReference>() {}); + return data; } private transient Dao dataAccess; - public ParticipantData() {} - public ParticipantData(Dao dao) { dataAccess = dao; } @@ -97,25 +74,11 @@ public ParticipantData(long participantDataId, String ddpParticipantId, int ddpI this.ddpParticipantId = ddpParticipantId; this.ddpInstanceId = ddpInstanceId; this.fieldTypeId = fieldTypeId; - setData(data); + this.data = data; } public static ParticipantData parseDto(@NonNull org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData participantData) { - - class LocalParticipantData extends ParticipantData { - - @JsonProperty("data") - private Map data; - - public LocalParticipantData(long participantDataId, String ddpParticipantId, int ddpInstanceId, String fieldTypeId, Map data) { - this.participantDataId = participantDataId; - this.ddpParticipantId = ddpParticipantId; - this.ddpInstanceId = ddpInstanceId; - this.fieldTypeId = fieldTypeId; - this.data = data; - } - } - return new LocalParticipantData( + return new ParticipantData( participantData.getParticipantDataId(), participantData.getDdpParticipantId().orElse(StringUtils.EMPTY), participantData.getDdpInstanceId(), @@ -154,7 +117,7 @@ public void setData(String ddpParticipantId, int ddpInstanceId, String fieldType this.ddpParticipantId = ddpParticipantId; this.ddpInstanceId = ddpInstanceId; this.fieldTypeId = fieldTypeId; - setData(data); + this.data = data; } public long insertParticipantData(String userEmail) { @@ -164,7 +127,7 @@ public long insertParticipantData(String userEmail) { .withDdpParticipantId(this.ddpParticipantId) .withDdpInstanceId(this.ddpInstanceId) .withFieldTypeId(this.fieldTypeId) - .withData(this.data) + .withData(ObjectMapperSingleton.writeValueAsString(this.data)) .withLastChanged(System.currentTimeMillis()) .withChangedBy(userEmail) .build(); @@ -206,42 +169,11 @@ String getRelationshipId() { return this.getData().getOrDefault(FamilyMemberConstants.RELATIONSHIP_ID, null); } - public boolean updateParticipantData(int participantDataId, String changedByUser) { - org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData participantData = new org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData.Builder() - .withParticipantDataId(participantDataId) - .withDdpParticipantId(this.ddpParticipantId) - .withDdpInstanceId(this.ddpInstanceId) - .withFieldTypeId(this.fieldTypeId) - .withData(this.data) - .withLastChanged(System.currentTimeMillis()) - .withChangedBy(changedByUser) - .build(); - - int rowsAffected = ((ParticipantDataDao) dataAccess).updateParticipantDataColumn(participantData); - - DDPInstanceDto ddpInstanceDto = new DDPInstanceDao().getDDPInstanceByInstanceId(ddpInstanceId).orElseThrow(); - - UpsertPainlessFacade.of(DBConstants.DDP_PARTICIPANT_DATA_ALIAS, participantData, ddpInstanceDto, "participantDataId", "participantDataId", participantDataId) - .export(); - - return rowsAffected == 1; - } - - private void exportToES(int participantDataId, org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData participantData, String uniqueIdentifier) { - DDPInstanceDto ddpInstanceDto = ((DDPInstanceDao) setDataAccess(new DDPInstanceDao())).getDDPInstanceByInstanceId(ddpInstanceId).orElseThrow(); - - Generator generator = new ParamsGenerator(participantData, ddpInstanceDto.getInstanceName()); - ScriptBuilder scriptBuilder = new NestedScriptBuilder("participantData", uniqueIdentifier); - MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("participantDataId", participantDataId); - NestedQueryBuilder nestedQueryBuilder = new NestedQueryBuilder(String.join(".", ESObjectConstants.DSM, generator.getPropertyName()), matchQueryBuilder, ScoreMode.Avg); - UpsertPainless upsertPainless = new UpsertPainless(generator, ddpInstanceDto.getEsParticipantIndex(), scriptBuilder, nestedQueryBuilder); - upsertPainless.export(); - } - public Optional findProband(List participantDataList) { return Objects.requireNonNull(participantDataList).stream() .filter(participantDataDto -> { - Map pDataMap = GSON.fromJson(participantDataDto.getData().orElse(StringUtils.EMPTY), Map.class); + Map pDataMap = ObjectMapperSingleton.readValue(participantDataDto.getData().orElse(StringUtils.EMPTY), new TypeReference>() {}); return FamilyMemberConstants.MEMBER_TYPE_SELF.equals(pDataMap.get(FamilyMemberConstants.MEMBER_TYPE)); }) .findFirst(); From 6c85b1ab2e03c4e66742693a43a3e648a555ed06 Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 1 Feb 2022 18:59:35 +0400 Subject: [PATCH 370/405] DDP-7443 | remove unused imports and code, minor refactoring --- .../painless/NestedUpsertPainlessFacade.java | 3 +-- .../model/participant/ParticipantWrapper.java | 18 ++---------------- .../participant/data/ParticipantData.java | 6 ++++++ 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedUpsertPainlessFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedUpsertPainlessFacade.java index 132c1616e..6ff84ab51 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedUpsertPainlessFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedUpsertPainlessFacade.java @@ -24,7 +24,6 @@ protected ScriptBuilder buildScriptBuilder() { protected QueryBuilder buildQueryBuilder() { MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder(fieldName, fieldValue); if (ESObjectConstants.DOC_ID.equals(fieldName)) return matchQueryBuilder; - return new NestedQueryBuilder(String.join(DBConstants.ALIAS_DELIMITER, ESObjectConstants.DSM, generator.getPropertyName(), - fieldName), matchQueryBuilder, ScoreMode.Avg); + return new NestedQueryBuilder(String.join(DBConstants.ALIAS_DELIMITER, ESObjectConstants.DSM, generator.getPropertyName()), matchQueryBuilder, ScoreMode.Avg); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java index ed20984bf..4bfe257ad 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java @@ -17,6 +17,7 @@ import org.broadinstitute.dsm.model.elastic.search.ElasticSearchable; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import org.broadinstitute.dsm.statics.DBConstants; +import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.elasticsearch.index.query.BoolQueryBuilder; import org.slf4j.Logger; @@ -71,11 +72,6 @@ public ParticipantWrapperResult getFilteredList() { }) .orElseGet(() -> { fetchAndPrepareData(ddpInstance); - //if study is AT -// if ("atcp".equals(ddpInstance.getName())) { -// DefaultValues defaultValues = new DefaultValues(participantData, esData.getEsParticipants(), ddpInstance, null); -// participantData = defaultValues.addDefaultValues(); -// } sortBySelfElseById(participantData.values()); return new ParticipantWrapperResult(esData.getTotalCount(), collectData(ddpInstance)); }); @@ -93,17 +89,7 @@ private void fetchAndPrepareDataByFilters(DDPInstance ddpInstance, Map data; + public ParticipantData() { + + } + public void setData(Map data) { this.data = data; } @@ -62,6 +67,7 @@ public Map getData() { return data; } + @JsonIgnore private transient Dao dataAccess; public ParticipantData(Dao dao) { From 9c53e2dbb24ffb28796b685f997c7d12cf59cfbf Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 2 Feb 2022 12:02:47 +0400 Subject: [PATCH 371/405] DDP-7443 | fix matchquerybuilder by giving full path of field in case of nested type --- .../export/painless/NestedUpsertPainlessFacade.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedUpsertPainlessFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedUpsertPainlessFacade.java index 6ff84ab51..ed989e20c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedUpsertPainlessFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/NestedUpsertPainlessFacade.java @@ -22,8 +22,14 @@ protected ScriptBuilder buildScriptBuilder() { @Override protected QueryBuilder buildQueryBuilder() { - MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder(fieldName, fieldValue); - if (ESObjectConstants.DOC_ID.equals(fieldName)) return matchQueryBuilder; - return new NestedQueryBuilder(String.join(DBConstants.ALIAS_DELIMITER, ESObjectConstants.DSM, generator.getPropertyName()), matchQueryBuilder, ScoreMode.Avg); + MatchQueryBuilder matchQueryBuilder; + if (ESObjectConstants.DOC_ID.equals(fieldName)) { + matchQueryBuilder = new MatchQueryBuilder(fieldName, fieldValue); + return matchQueryBuilder; + } else { + String path = String.join(DBConstants.ALIAS_DELIMITER, ESObjectConstants.DSM, generator.getPropertyName()); + matchQueryBuilder = new MatchQueryBuilder(String.join(DBConstants.ALIAS_DELIMITER, path, fieldName), fieldValue); + return new NestedQueryBuilder(path, matchQueryBuilder, ScoreMode.Avg); + } } } From 4ef27635245c95546b56c52d4045e20daeb6cc61 Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 2 Feb 2022 15:15:23 +0400 Subject: [PATCH 372/405] DDP-7443 | parse dsmKitRequestId to Long and add .export() call in the KitStatusChangeRoute --- .../java/org/broadinstitute/dsm/db/KitRequestShipping.java | 2 +- .../org/broadinstitute/dsm/route/KitStatusChangeRoute.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java index 3447f1316..bce812f68 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java @@ -1442,7 +1442,7 @@ public static void reactivateKitRequest(@NonNull String dsmKitRequestId, String long dsmKitId = KitRequestShipping.writeNewKit(dsmKitRequestId, kitRequestShipping.getEasypostAddressId(), message, false); kitRequestShipping.setDsmKitId(dsmKitId); - UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, "dsmKitId", "dsmKitRequestId", dsmKitRequestId) + UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, "dsmKitId", "dsmKitRequestId", Long.parseLong(dsmKitRequestId)) .export(); } } diff --git a/src/main/java/org/broadinstitute/dsm/route/KitStatusChangeRoute.java b/src/main/java/org/broadinstitute/dsm/route/KitStatusChangeRoute.java index 5acb2cfb7..24e28bfd0 100644 --- a/src/main/java/org/broadinstitute/dsm/route/KitStatusChangeRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/KitStatusChangeRoute.java @@ -185,7 +185,8 @@ else if (RoutePath.RECEIVED_KIT_REQUEST.equals(changeType)) { if (kitDDPNotification != null) { EventUtil.triggerDDP(conn, kitDDPNotification); } - UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, "ddpLabel", "ddpLabel", kit); + UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, "ddpLabel", "ddpLabel", kit) + .export(); } else if (RoutePath.TRACKING_SCAN_REQUEST.equals(changeType)) { logger.info("Added tracking for kit w/ kit_label " + kit); From 65693b74f06c7b163608ebf91fe836806999def2 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 2 Feb 2022 16:34:03 +0400 Subject: [PATCH 373/405] DDP-7121 | applying comment changes --- .../broadinstitute/dsm/db/Participant.java | 9 ++++----- .../dsm/db/structure/DBElement.java | 2 -- .../dsm/model/AbstractionWrapper.java | 19 +++++++++++-------- .../export/ElasticMappingExportAdapter.java | 9 +++++---- 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/Participant.java b/src/main/java/org/broadinstitute/dsm/db/Participant.java index 0aa335630..35a1eb8aa 100644 --- a/src/main/java/org/broadinstitute/dsm/db/Participant.java +++ b/src/main/java/org/broadinstitute/dsm/db/Participant.java @@ -56,7 +56,7 @@ public class Participant { primaryKey = DBConstants.PARTICIPANT_ID, columnPrefix = "") @ColumnName (DBConstants.ASSIGNEE_ID_MR) - private long assigneeIdMr; + private String assigneeIdMr; @TableName ( name = DBConstants.DDP_PARTICIPANT, @@ -64,7 +64,7 @@ public class Participant { primaryKey = DBConstants.PARTICIPANT_ID, columnPrefix = "") @ColumnName (DBConstants.ASSIGNEE_ID_TISSUE) - private long assigneeIdTissue; + private String assigneeIdTissue; private String realm; @TableName ( @@ -154,7 +154,7 @@ public Map getDynamicFields() { public Participant() {} - public Participant(long participantId, String ddpParticipantId, long assigneeIdMr, long assigneeIdTissue, String instanceName, + public Participant(long participantId, String ddpParticipantId, String assigneeIdMr, String assigneeIdTissue, String instanceName, String createdOncHistory, String reviewedOncHistory, String crSent, String crReceived, String notes, boolean minimalMr, boolean abstractionReady, String additionalValuesJson, long exitDate) { this.participantId = participantId; @@ -188,8 +188,7 @@ public static Participant getParticipant(@NonNull Map assignee } Participant participant = new Participant(rs.getLong(DBConstants.PARTICIPANT_ID), rs.getString(DBConstants.DDP_PARTICIPANT_ID), - rs.getLong(DBConstants.ASSIGNEE_ID_MR), - rs.getLong(DBConstants.ASSIGNEE_ID_TISSUE), realm, + assigneeMR, assigneeTissue, realm, rs.getString(DBConstants.ONC_HISTORY_CREATED), rs.getString(DBConstants.ONC_HISTORY_REVIEWED), rs.getString(DBConstants.CR_SENT), diff --git a/src/main/java/org/broadinstitute/dsm/db/structure/DBElement.java b/src/main/java/org/broadinstitute/dsm/db/structure/DBElement.java index d27f46456..d33a41393 100644 --- a/src/main/java/org/broadinstitute/dsm/db/structure/DBElement.java +++ b/src/main/java/org/broadinstitute/dsm/db/structure/DBElement.java @@ -1,10 +1,8 @@ package org.broadinstitute.dsm.db.structure; -import java.text.SimpleDateFormat; import java.time.Instant; import java.time.ZoneId; import java.time.format.DateTimeFormatter; -import java.util.List; import lombok.Data; diff --git a/src/main/java/org/broadinstitute/dsm/model/AbstractionWrapper.java b/src/main/java/org/broadinstitute/dsm/model/AbstractionWrapper.java index 16c1e190e..8007d05bb 100644 --- a/src/main/java/org/broadinstitute/dsm/model/AbstractionWrapper.java +++ b/src/main/java/org/broadinstitute/dsm/model/AbstractionWrapper.java @@ -1,5 +1,16 @@ package org.broadinstitute.dsm.model; +import static org.broadinstitute.ddp.db.TransactionWrapper.inTransaction; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; + import com.google.gson.Gson; import lombok.Data; import lombok.NonNull; @@ -14,14 +25,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.*; - -import static org.broadinstitute.ddp.db.TransactionWrapper.inTransaction; - @Data public class AbstractionWrapper { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticMappingExportAdapter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticMappingExportAdapter.java index 88bab66b0..7f7506e54 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticMappingExportAdapter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ElasticMappingExportAdapter.java @@ -13,14 +13,15 @@ public class ElasticMappingExportAdapter extends BaseExporter { @Override public void export() { - logger.info("initialize exporting mapping to ES"); - PutMappingRequest putMappingRequest = new PutMappingRequest(requestPayload.getIndex()); + String esIndex = requestPayload.getIndex(); + logger.info("initialize exporting mapping to ES with index: " + esIndex); + PutMappingRequest putMappingRequest = new PutMappingRequest(esIndex); putMappingRequest.source(source); try { clientInstance.indices().putMapping(putMappingRequest, RequestOptions.DEFAULT); } catch (IOException e) { - throw new RuntimeException("Error occurred while updating mapping to ES", e); + throw new RuntimeException("Error occurred while updating mapping to ES with index: " + esIndex, e); } - logger.info("successfully exported mapping to ES"); + logger.info("successfully exported mapping to ES with index: " + esIndex); } } From a051c2c884df4aa93b9028ba360a7ad8b187b5be Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 2 Feb 2022 16:44:04 +0400 Subject: [PATCH 374/405] DDP-7121 | extract DateSplitterHelper.splitter() for Date splitting --- .../dsm/model/elastic/export/ExportFacade.java | 2 +- .../elastic/filter/splitter/DateGreaterSplitter.java | 11 ++++++++--- .../elastic/filter/splitter/DateLowerSplitter.java | 5 +---- .../filter/splitter/GreaterThanEqualsSplitter.java | 2 -- .../broadinstitute/dsm/model/elastic/UtilTest.java | 4 ++-- .../elastic/export/process/SingleProcessorTest.java | 4 ++-- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java index ddad579ed..fc208b947 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/ExportFacade.java @@ -106,7 +106,7 @@ private BaseGenerator.PropertyInfo getPropertyInfo() { private void upsertData(Map elasticDataToExport) { RequestPayload requestPayload = new RequestPayload(exportFacadePayload.getIndex(), exportFacadePayload.getDocId()); - logger.info("Built upsert data request payload"); + logger.info("Built upsert data request payload for the index " + exportFacadePayload.getIndex()); exportable = new ElasticDataExportAdapter(); exportable.setRequestPayload(requestPayload); exportable.setSource(elasticDataToExport); diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateGreaterSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateGreaterSplitter.java index 8771e98e4..a26bd3d6d 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateGreaterSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateGreaterSplitter.java @@ -8,10 +8,15 @@ public class DateGreaterSplitter extends GreaterThanEqualsSplitter { @Override public String[] getValue() { - return new String[]{splittedFilter[1].split(Filter.DATE_FORMAT)[1] + return DateSplitterHelper.splitter(splittedFilter[1]); + } +} + +class DateSplitterHelper { + public static String[] splitter(String filter) { + return new String[]{filter.split(Filter.DATE_FORMAT)[1] .replace(Filter.OPEN_PARENTHESIS, StringUtils.EMPTY) .replace(Filter.CLOSE_PARENTHESIS, StringUtils.EMPTY) .split(Util.COMMA_SEPARATOR)[0]}; - } -} +} \ No newline at end of file diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateLowerSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateLowerSplitter.java index bb08477ea..2cfe7ed2f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateLowerSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateLowerSplitter.java @@ -9,9 +9,6 @@ public class DateLowerSplitter extends LessThanEqualsSplitter { @Override public String[] getValue() { // STR_TO_DATE('2012-01-01', %yyyy-%MM-%dd) - return new String[]{splittedFilter[1].split(Filter.DATE_FORMAT)[1] - .replace(Filter.OPEN_PARENTHESIS, StringUtils.EMPTY) - .replace(Filter.CLOSE_PARENTHESIS, StringUtils.EMPTY) - .split(Util.COMMA_SEPARATOR)[0]}; + return DateSplitterHelper.splitter(splittedFilter[1]); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/GreaterThanEqualsSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/GreaterThanEqualsSplitter.java index ba5d07a0c..f35c9f30e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/GreaterThanEqualsSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/GreaterThanEqualsSplitter.java @@ -5,8 +5,6 @@ public class GreaterThanEqualsSplitter extends BaseSplitter { - Operator greaterThanEquals = Operator.GREATER_THAN_EQUALS; - @Override public String[] split() { return filter.split(Filter.LARGER_EQUALS_TRIMMED); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java index f1663a719..eee2623cf 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java @@ -39,8 +39,8 @@ public void underscoresToCamelCase() { @Test public void transformObjectToMap() { Participant participant = new Participant( - 1, "QWERTY", 0, - 0, "instance", "2020-10-28", + 1, "QWERTY", null, + null, "instance", "2020-10-28", "2020-10-28", "2020-10-28", "2020-10-28", "ptNotes", true, true, "additionalValuesJson", 1934283746283L); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/SingleProcessorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/SingleProcessorTest.java index 7b9dd426d..8a247c9ab 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/SingleProcessorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/process/SingleProcessorTest.java @@ -28,7 +28,7 @@ public void processExisting() { generator.setPayload(new GeneratorPayload(new NameValue("p.additionalValuesJson", "{\"key\":\"value\"}"), 0)); ESDsm esDsm = new ESDsm(); - esDsm.setParticipant(new Participant(2174, null, 0, 0, null, null, null, + esDsm.setParticipant(new Participant(2174, null, null, null, null, null, null, null, null, null, false, false, "{\"key\": \"oldVal\"}", 12874512387612L )); BaseProcessor processor = new SingleProcessor(); processor.setEsDsm(esDsm); @@ -49,7 +49,7 @@ public void processNew() { generator.setPayload(new GeneratorPayload(new NameValue("p.additionalValuesJson", "{\"key\":\"value\"}"), 0)); ESDsm esDsm = new ESDsm(); - esDsm.setParticipant(new Participant(2174, null, 0, 0, null, null, null, + esDsm.setParticipant(new Participant(2174, null, null, null, null, null, null, null, null, null, false, false, "", 12874512387612L )); BaseProcessor processor = new SingleProcessor(); processor.setEsDsm(esDsm); From 0a7c51131e3a0558a91e61be5f3a790b199b8f5b Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 3 Feb 2022 10:45:45 +0400 Subject: [PATCH 375/405] DDP-7443 | applying changes according to the comments --- .../org/broadinstitute/dsm/db/KitRequestCreateLabel.java | 4 ++-- .../org/broadinstitute/dsm/db/KitRequestShipping.java | 3 ++- src/main/java/org/broadinstitute/dsm/model/BSPKit.java | 5 +++-- .../broadinstitute/dsm/model/at/ReceiveKitRequest.java | 4 +++- .../defaultvalues/RgpAutomaticProbandDataCreator.java | 3 ++- .../model/elastic/export/painless/ParamsGenerator.java | 1 - .../dsm/model/familymember/AddFamilyMember.java | 2 +- .../dsm/model/participant/ParticipantWrapper.java | 1 - .../java/org/broadinstitute/dsm/route/KitLabelRoute.java | 2 +- .../org/broadinstitute/dsm/statics/ESObjectConstants.java | 3 +++ .../dsm/util/DDPMedicalRecordDataRequest.java | 8 ++++---- .../org/broadinstitute/dsm/util/MedicalRecordUtil.java | 7 +++---- src/main/java/org/broadinstitute/dsm/util/SystemUtil.java | 1 + src/test/java/org/broadinstitute/dsm/RouteTestSample.java | 3 ++- .../dsm/util/tools/MedicalRecordMigrationTool.java | 3 ++- .../dsm/util/tools/TissueDataMigrationTool.java | 3 ++- 16 files changed, 31 insertions(+), 22 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/KitRequestCreateLabel.java b/src/main/java/org/broadinstitute/dsm/db/KitRequestCreateLabel.java index 99db45363..fe75a5a0b 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitRequestCreateLabel.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitRequestCreateLabel.java @@ -81,7 +81,8 @@ public static void updateKitLabelRequested(KitRequestShipping[] kitRequests, @No * Select all sample requests without kit per realm * optional per kit type */ - public static void updateKitLabelRequested(String realm, String type, @NonNull String userId) { + public static void updateKitLabelRequested(String realm, String type, @NonNull String userId, + DDPInstanceDto ddpInstanceDto) { List kitsNoLabel = new ArrayList<>(); SimpleResult results = inTransaction((conn) -> { SimpleResult dbVals = new SimpleResult(); @@ -111,7 +112,6 @@ public static void updateKitLabelRequested(String realm, String type, @NonNull S throw new RuntimeException("Error looking up the latestKitRequests ", results.resultException); } logger.info("Found " + kitsNoLabel.size() + " kit requests which need a label"); - DDPInstanceDto ddpInstanceDto = new DDPInstanceDao().getDDPInstanceByInstanceName(realm).orElseThrow(); if (!kitsNoLabel.isEmpty()) { for (KitRequestCreateLabel uploadedKit : kitsNoLabel) { KitRequestShipping.updateKit(Long.parseLong(uploadedKit.getDsmKitId()), userId, ddpInstanceDto); diff --git a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java index bce812f68..b6222701f 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java @@ -1442,7 +1442,8 @@ public static void reactivateKitRequest(@NonNull String dsmKitRequestId, String long dsmKitId = KitRequestShipping.writeNewKit(dsmKitRequestId, kitRequestShipping.getEasypostAddressId(), message, false); kitRequestShipping.setDsmKitId(dsmKitId); - UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, "dsmKitId", "dsmKitRequestId", Long.parseLong(dsmKitRequestId)) + UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, ESObjectConstants.DSM_KIT_ID, + ESObjectConstants.DSM_KIT_REQUEST_ID, Long.parseLong(dsmKitRequestId)) .export(); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/BSPKit.java b/src/main/java/org/broadinstitute/dsm/model/BSPKit.java index 1f4df273e..0bfc9a09a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/BSPKit.java +++ b/src/main/java/org/broadinstitute/dsm/model/BSPKit.java @@ -84,7 +84,7 @@ public Optional receiveBSPKit(String kitLabel, NotificationUtil noti if (StringUtils.isBlank(maybeBspKitQueryResult.getDdpParticipantId())) { throw new RuntimeException("No participant id for " + kitLabel + " from " + maybeBspKitQueryResult.getInstanceName()); } - logger.info("particpant id is " + maybeBspKitQueryResult.getDdpParticipantId()); + logger.info("participant id is " + maybeBspKitQueryResult.getDdpParticipantId()); DDPInstance ddpInstance = DDPInstance.getDDPInstance(maybeBspKitQueryResult.getInstanceName()); InstanceSettings instanceSettings = new InstanceSettings(); InstanceSettingsDto instanceSettingsDto = instanceSettings.getInstanceSettings(maybeBspKitQueryResult.getInstanceName()); @@ -153,7 +153,8 @@ private void updateKitAndExport(String kitLabel, BSPKitDao bspKitDao, BSPKitDto DDPInstanceDto ddpInstanceDto = new DDPInstanceDao().getDDPInstanceByInstanceName(maybeBspKitQueryResult.getInstanceName()).orElseThrow(); - UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, "kitLabel", "kitLabel", kitLabel) + UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, ESObjectConstants.KIT_LABEL + , ESObjectConstants.KIT_LABEL, kitLabel) .export(); } diff --git a/src/main/java/org/broadinstitute/dsm/model/at/ReceiveKitRequest.java b/src/main/java/org/broadinstitute/dsm/model/at/ReceiveKitRequest.java index ce4f80548..b8472d21a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/at/ReceiveKitRequest.java +++ b/src/main/java/org/broadinstitute/dsm/model/at/ReceiveKitRequest.java @@ -11,6 +11,7 @@ import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData; import org.broadinstitute.dsm.model.elastic.export.painless.UpsertPainlessFacade; import org.broadinstitute.dsm.statics.DBConstants; +import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.NotificationUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -91,7 +92,8 @@ private static boolean updateData(@NonNull String data, @NonNull long participan .withDdpInstanceId(ddpInstanceDto.getDdpInstanceId()) .build(); - UpsertPainlessFacade.of(DBConstants.DDP_PARTICIPANT_DATA_ALIAS, participantData, ddpInstanceDto, "participantDataId", "participantDataId", participantDataId) + UpsertPainlessFacade.of(DBConstants.DDP_PARTICIPANT_DATA_ALIAS, participantData, ddpInstanceDto, ESObjectConstants.PARTICIPANT_DATA_ID, + ESObjectConstants.PARTICIPANT_DATA_ID, participantDataId) .export(); return true; diff --git a/src/main/java/org/broadinstitute/dsm/model/defaultvalues/RgpAutomaticProbandDataCreator.java b/src/main/java/org/broadinstitute/dsm/model/defaultvalues/RgpAutomaticProbandDataCreator.java index 2ebcdbb31..b71ca01b6 100644 --- a/src/main/java/org/broadinstitute/dsm/model/defaultvalues/RgpAutomaticProbandDataCreator.java +++ b/src/main/java/org/broadinstitute/dsm/model/defaultvalues/RgpAutomaticProbandDataCreator.java @@ -22,6 +22,7 @@ import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; +import org.broadinstitute.dsm.util.SystemUtil; public class RgpAutomaticProbandDataCreator extends BasicDefaultDataMaker { @@ -54,7 +55,7 @@ protected boolean setDefaultData() { probandDataMap ); participantData.addDefaultOptionsValueToData(columnsWithDefaultOptions); - participantData.insertParticipantData("SYSTEM"); + participantData.insertParticipantData(SystemUtil.SYSTEM); columnsWithDefaultOptionsFilteredByElasticExportWorkflow.forEach((col, val) -> ElasticSearchUtil.writeWorkflow(WorkflowForES.createInstanceWithStudySpecificData(instance, participantId, col, val, diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGenerator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGenerator.java index d0a99b081..e77f75f65 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGenerator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGenerator.java @@ -9,7 +9,6 @@ public class ParamsGenerator implements Generator { - static final String PARAMS = "params"; private Object source; private String realm; diff --git a/src/main/java/org/broadinstitute/dsm/model/familymember/AddFamilyMember.java b/src/main/java/org/broadinstitute/dsm/model/familymember/AddFamilyMember.java index dacb2b803..e51982566 100644 --- a/src/main/java/org/broadinstitute/dsm/model/familymember/AddFamilyMember.java +++ b/src/main/java/org/broadinstitute/dsm/model/familymember/AddFamilyMember.java @@ -82,7 +82,7 @@ protected void copyProbandData() { Map participantDataData = participantData.getData(); if (Objects.isNull(participantDataData)) throw new NoSuchElementException(); List participantDataByParticipantId = - participantData.getParticipantDataByParticipantId(addFamilyMemberPayload.getParticipantId().orElse("")); + participantData.getParticipantDataByParticipantId(addFamilyMemberPayload.getParticipantId().orElse(StringUtils.EMPTY)); Optional maybeProbandData = participantData.findProband(participantDataByParticipantId); Optional maybeParticipantData = maybeProbandData.map(org.broadinstitute.dsm.model.participant.data.ParticipantData::parseDto); maybeParticipantData.ifPresent(participantData -> participantData.getData().forEach(participantDataData::putIfAbsent)); diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java index 4bfe257ad..57dcde175 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java @@ -77,7 +77,6 @@ public ParticipantWrapperResult getFilteredList() { }); } - //TODO could be better, good place for refactoring for future private void fetchAndPrepareDataByFilters(DDPInstance ddpInstance, Map filters) { FilterParser parser = new FilterParser(); BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); diff --git a/src/main/java/org/broadinstitute/dsm/route/KitLabelRoute.java b/src/main/java/org/broadinstitute/dsm/route/KitLabelRoute.java index 493a5d6b8..11bb7681f 100644 --- a/src/main/java/org/broadinstitute/dsm/route/KitLabelRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/KitLabelRoute.java @@ -51,7 +51,7 @@ public Object processRequest(Request request, Response response, String userId) if (queryParams.value(RoutePath.KIT_TYPE) != null) { kitType = queryParams.get(RoutePath.KIT_TYPE).value(); } - KitRequestCreateLabel.updateKitLabelRequested(realm, kitType, userIdRequest); + KitRequestCreateLabel.updateKitLabelRequested(realm, kitType, userIdRequest, ddpInstanceDto); return new Result(200); } else { diff --git a/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java b/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java index 13173903c..e1d51725c 100644 --- a/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java +++ b/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java @@ -38,6 +38,9 @@ public class ESObjectConstants { // kit public static final String KIT_REQUEST_SHIPPING = "kitRequestShipping"; + public static final String DSM_KIT_ID = "dsmKitId"; + public static final String DSM_KIT_REQUEST_ID = "dsmKitRequestId"; + public static final String KIT_LABEL = "kitLabel"; //common public static final String DDP_PARTICIPANT_ID = "ddpParticipantId"; diff --git a/src/main/java/org/broadinstitute/dsm/util/DDPMedicalRecordDataRequest.java b/src/main/java/org/broadinstitute/dsm/util/DDPMedicalRecordDataRequest.java index f3bafda9a..6ff6cab00 100644 --- a/src/main/java/org/broadinstitute/dsm/util/DDPMedicalRecordDataRequest.java +++ b/src/main/java/org/broadinstitute/dsm/util/DDPMedicalRecordDataRequest.java @@ -94,7 +94,7 @@ public void writeParticipantIntoDb(@NonNull Connection conn, @NonNull String ins if (MedicalRecordUtil.isParticipantInDB(conn, institutionRequest.getParticipantId(), instanceId)) { //participant already exists if (MedicalRecordUtil.updateParticipant(conn, institutionRequest.getParticipantId(), instanceId, - institutionRequest.getId(), institutionRequest.getLastUpdated(), MedicalRecordUtil.SYSTEM)) { + institutionRequest.getId(), institutionRequest.getLastUpdated(), SystemUtil.SYSTEM)) { writeInstitutionInfo(conn, institutionRequest, instanceId); //participant lastVersion changed Collection medicalRecordIds = getMedicalRecordIds(conn, institutionRequest.getParticipantId(), instanceId); @@ -111,7 +111,7 @@ public void writeParticipantIntoDb(@NonNull Connection conn, @NonNull String ins .withDdpParticipantId(institutionRequest.getParticipantId()) .withLastVersion(institutionRequest.getId()) .withLastVersionDate(institutionRequest.getLastUpdated()) - .withChangedBy(MedicalRecordUtil.SYSTEM) + .withChangedBy(SystemUtil.SYSTEM) .build(); new ParticipantDao().create(participantDto); writeInstitutionInfo(conn, institutionRequest, instanceId); @@ -126,7 +126,7 @@ public void writePhysiciansIntoDb(@NonNull Connection conn, @NonNull String inst .withDdpParticipantId(participantId) .withLastVersion(0) .withLastVersionDate(ptLastUpdated) - .withChangedBy(MedicalRecordUtil.SYSTEM) + .withChangedBy(SystemUtil.SYSTEM) .build(); new ParticipantDao().create(participantDto); MedicalRecordUtil.writeNewRecordIntoDb(conn, SQL_INSERT_ONC_HISTORY, participantId, instanceId); @@ -136,7 +136,7 @@ public void writePhysiciansIntoDb(@NonNull Connection conn, @NonNull String inst //pt already exists Number lastVersion = MedicalRecordUtil.getParticipantLastVersion(conn, participantId, instanceId); long newVersion = lastVersion.longValue() + 1; - MedicalRecordUtil.updateParticipant(conn, participantId, instanceId, newVersion, ptLastUpdated, MedicalRecordUtil.SYSTEM); + MedicalRecordUtil.updateParticipant(conn, participantId, instanceId, newVersion, ptLastUpdated, SystemUtil.SYSTEM); } //new physician/institution Number medicalRecordId = MedicalRecordUtil.isInstitutionInDB(conn, participantId, institutionId, instanceId, type); diff --git a/src/main/java/org/broadinstitute/dsm/util/MedicalRecordUtil.java b/src/main/java/org/broadinstitute/dsm/util/MedicalRecordUtil.java index dae0e21cb..c4448a53a 100644 --- a/src/main/java/org/broadinstitute/dsm/util/MedicalRecordUtil.java +++ b/src/main/java/org/broadinstitute/dsm/util/MedicalRecordUtil.java @@ -29,7 +29,6 @@ public class MedicalRecordUtil { private static final String SQL_SELECT_MEDICAL_RECORD_ID_AND_TYPE_FOR_PARTICIPANT = "SELECT rec.medical_record_id, inst.type FROM ddp_institution inst, ddp_participant part, ddp_medical_record rec " + "WHERE part.participant_id = inst.participant_id AND rec.institution_id = inst.institution_id AND NOT rec.deleted <=> 1 AND part.participant_id = ? AND inst.type = ?"; - public static final String SYSTEM = "SYSTEM"; public static final String NOT_SPECIFIED = "NOT_SPECIFIED"; public static final String OTHER = "OTHER"; @@ -38,7 +37,7 @@ public static void writeNewMedicalRecordIntoDb(Connection conn, String query, St try (PreparedStatement insertNewRecord = conn.prepareStatement(query)) { insertNewRecord.setString(1, id); insertNewRecord.setLong(2, System.currentTimeMillis()); - insertNewRecord.setString(3, SYSTEM); + insertNewRecord.setString(3, SystemUtil.SYSTEM); int result = insertNewRecord.executeUpdate(); if (result > 1) { // 0 or 1 is good throw new RuntimeException("Error updating row"); @@ -61,9 +60,9 @@ public static void writeNewRecordIntoDb(Connection conn, String query, String id insertNewRecord.setString(1, id); insertNewRecord.setString(2, instanceId); insertNewRecord.setLong(3, currentMilli); - insertNewRecord.setString(4, SYSTEM); + insertNewRecord.setString(4, SystemUtil.SYSTEM); insertNewRecord.setLong(5, currentMilli); - insertNewRecord.setString(6, SYSTEM); + insertNewRecord.setString(6, SystemUtil.SYSTEM); int result = insertNewRecord.executeUpdate(); // 1 (inserted) or 2 (updated) is good if (result == 2) { diff --git a/src/main/java/org/broadinstitute/dsm/util/SystemUtil.java b/src/main/java/org/broadinstitute/dsm/util/SystemUtil.java index 831b125f1..32b4eb951 100644 --- a/src/main/java/org/broadinstitute/dsm/util/SystemUtil.java +++ b/src/main/java/org/broadinstitute/dsm/util/SystemUtil.java @@ -34,6 +34,7 @@ public class SystemUtil { public static final long MILLIS_PER_DAY = 1000 * 60 * 60 * 24; public static final long MILLIS_PER_HOUR = 1000 * 60 * 60; + public static final String SYSTEM = "SYSTEM"; private static final String LINEBREAK_UNIVERSAL = "\n"; private static final String LINEBREAK = "\r"; diff --git a/src/test/java/org/broadinstitute/dsm/RouteTestSample.java b/src/test/java/org/broadinstitute/dsm/RouteTestSample.java index d89d35e67..995246d9f 100644 --- a/src/test/java/org/broadinstitute/dsm/RouteTestSample.java +++ b/src/test/java/org/broadinstitute/dsm/RouteTestSample.java @@ -9,6 +9,7 @@ import org.apache.http.client.fluent.Request; import org.broadinstitute.ddp.util.GoogleBucket; import org.broadinstitute.dsm.db.*; +import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; import org.broadinstitute.dsm.exception.RateNotAvailableException; import org.broadinstitute.dsm.model.*; import org.broadinstitute.dsm.model.KitType; @@ -553,7 +554,7 @@ public void differentReturnPhoneAddress() throws Exception { public static void triggerLabelCreationAndWaitForLabel(String ddp, String kitType, long waitSeconds) { // set all kits of the realm to generate a label - KitRequestCreateLabel.updateKitLabelRequested(ddp, kitType, "1"); + KitRequestCreateLabel.updateKitLabelRequested(ddp, kitType, "1", new DDPInstanceDto.Builder().build()); //start method of label job List kitsLabelTriggered = KitUtil.getListOfKitsLabelTriggered(); if (!kitsLabelTriggered.isEmpty()) { diff --git a/src/test/java/org/broadinstitute/dsm/util/tools/MedicalRecordMigrationTool.java b/src/test/java/org/broadinstitute/dsm/util/tools/MedicalRecordMigrationTool.java index ec8864622..dfe00e8ca 100644 --- a/src/test/java/org/broadinstitute/dsm/util/tools/MedicalRecordMigrationTool.java +++ b/src/test/java/org/broadinstitute/dsm/util/tools/MedicalRecordMigrationTool.java @@ -11,6 +11,7 @@ import org.broadinstitute.dsm.util.DBTestUtil; import org.broadinstitute.dsm.util.DDPMedicalRecordDataRequest; import org.broadinstitute.dsm.util.MedicalRecordUtil; +import org.broadinstitute.dsm.util.SystemUtil; import org.broadinstitute.dsm.util.TestUtil; import org.broadinstitute.dsm.util.tools.util.DBUtil; import org.broadinstitute.dsm.util.tools.util.FileUtil; @@ -117,7 +118,7 @@ private static void migrate(@NonNull String realm, @NonNull String file) { .withDdpParticipantId(ddpParticipantId) .withLastVersion(0) .withLastVersionDate("MIGRATION_TOOL") - .withChangedBy(MedicalRecordUtil.SYSTEM) + .withChangedBy(SystemUtil.SYSTEM) .build(); new ParticipantDao().create(participantDto); MedicalRecordUtil.writeNewRecordIntoDb(conn, DDPMedicalRecordDataRequest.SQL_INSERT_ONC_HISTORY, diff --git a/src/test/java/org/broadinstitute/dsm/util/tools/TissueDataMigrationTool.java b/src/test/java/org/broadinstitute/dsm/util/tools/TissueDataMigrationTool.java index e43bcda69..de8e19c99 100644 --- a/src/test/java/org/broadinstitute/dsm/util/tools/TissueDataMigrationTool.java +++ b/src/test/java/org/broadinstitute/dsm/util/tools/TissueDataMigrationTool.java @@ -18,6 +18,7 @@ import org.broadinstitute.dsm.util.DBTestUtil; import org.broadinstitute.dsm.util.MedicalRecordUtil; import org.broadinstitute.dsm.util.PatchUtil; +import org.broadinstitute.dsm.util.SystemUtil; import org.broadinstitute.dsm.util.TestUtil; import org.broadinstitute.dsm.util.tools.util.DBUtil; import org.broadinstitute.dsm.util.tools.util.FileUtil; @@ -335,7 +336,7 @@ else if (StringUtils.isNotBlank(accessionNumber) && !accessionNumberInDB) { .withDdpParticipantId(ddpParticipantId) .withLastVersion(0) .withLastVersionDate("TISSUE_MIGRATION_TOOL") - .withChangedBy(MedicalRecordUtil.SYSTEM) + .withChangedBy(SystemUtil.SYSTEM) .build(); new ParticipantDao().create(participantDto); } From 68f73e2d32544b87716666302bd54888a7e5c96e Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 3 Feb 2022 10:55:20 +0400 Subject: [PATCH 376/405] DDP-7443 | incorporate suggestions into the code --- .../model/participant/ParticipantWrapper.java | 2 +- .../org/broadinstitute/dsm/util/KitUtil.java | 25 ++++++++++--------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java index 57dcde175..f71a36934 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java @@ -237,7 +237,7 @@ private void mapTissueToProperOncHistoryDetail(List oncHistory void sortBySelfElseById(Collection> participantDatas) { participantDatas.forEach(pDataList -> pDataList.sort((o1, o2) -> { - Map pData = new Gson().fromJson(o1.getData().orElse(""), new TypeToken>() {}.getType()); + Map pData = new Gson().fromJson(o1.getData().orElse(StringUtils.EMPTY), new TypeToken>() {}.getType()); if (Objects.nonNull(pData) && FamilyMemberConstants.MEMBER_TYPE_SELF.equals(pData.get(FamilyMemberConstants.MEMBER_TYPE))) return -1; return o1.getParticipantDataId() - o2.getParticipantDataId(); })); diff --git a/src/main/java/org/broadinstitute/dsm/util/KitUtil.java b/src/main/java/org/broadinstitute/dsm/util/KitUtil.java index 1a9a3be12..ffc2b6719 100644 --- a/src/main/java/org/broadinstitute/dsm/util/KitUtil.java +++ b/src/main/java/org/broadinstitute/dsm/util/KitUtil.java @@ -21,6 +21,7 @@ import org.broadinstitute.dsm.model.ddp.DDPParticipant; import org.broadinstitute.dsm.model.elastic.export.painless.UpsertPainlessFacade; import org.broadinstitute.dsm.statics.DBConstants; +import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.statics.QueryExtension; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -125,6 +126,7 @@ public HashMap getUnsentExpressKits(boolean showOnlyKitsWithNoEx public static void createLabel(List kitsLabelTriggered) { DBUtil.updateBookmark(System.currentTimeMillis(), BOOKMARK_LABEL_CREATION_RUNNING); + DDPInstanceDto ddpInstanceDto = null; for (KitRequestCreateLabel kitLabelTriggered : kitsLabelTriggered) { EasyPostUtil easyPostUtil = new EasyPostUtil(kitLabelTriggered.getInstanceName()); @@ -135,7 +137,7 @@ public static void createLabel(List kitsLabelTriggered) { DDPInstance ddpInstance = DDPInstance.getDDPInstance(kitLabelTriggered.getInstanceName()); //TODO -> before we finally switch to ddpInstanceDao/ddpInstanceDto pair - DDPInstanceDto ddpInstanceDto = new DDPInstanceDto.Builder() + ddpInstanceDto = new DDPInstanceDto.Builder() .withInstanceName(ddpInstance.getName()) .withEsParticipantIndex(ddpInstance.getParticipantIndexES()) .build(); @@ -153,7 +155,7 @@ public static void createLabel(List kitsLabelTriggered) { else { KitRequestShipping.deactivateKitRequest(Long.parseLong(kitLabelTriggered.getDsmKitRequestId()), "Participant not found", null, - "System", ddpInstanceDto); + SystemUtil.SYSTEM, ddpInstanceDto); logger.error("Didn't find participant " + kitLabelTriggered.getDdpParticipantId()); } } @@ -201,7 +203,7 @@ public static void createLabel(List kitsLabelTriggered) { } if (toAddress != null) { buyShipmentForKit(easyPostUtil, kitLabelTriggered.getDsmKitId(), kitLabelTriggered.getKitRequestSettings(), - kitLabelTriggered.getKitTyp(), toAddress.getId(), kitLabelTriggered.getBillingReference()); + kitLabelTriggered.getKitTyp(), toAddress.getId(), kitLabelTriggered.getBillingReference(), ddpInstanceDto); } } @@ -262,7 +264,7 @@ public static List getListOfKitsLabelTriggered() { } private static void buyShipmentForKit(@NonNull EasyPostUtil easyPostUtil, @NonNull String dsmKitId, @NonNull KitRequestSettings kitRequestSettings, - @NonNull KitType kitType, @NonNull String addressIdTo, String billingReference) { + @NonNull KitType kitType, @NonNull String addressIdTo, String billingReference, DDPInstanceDto ddpInstanceDto) { String errorMessage = ""; Shipment participantShipment = null; Shipment returnShipment = null; @@ -281,8 +283,6 @@ private static void buyShipmentForKit(@NonNull EasyPostUtil easyPostUtil, @NonNu catch (Exception e) { errorMessage += "Return: " + e.getMessage(); } - DDPInstanceDao ddpInstanceDao = new DDPInstanceDao(); - DDPInstanceDto ddpInstanceDto = ddpInstanceDao.getDDPInstanceByInstanceId(kitRequestSettings.getDdpInstanceId()).orElseThrow(); KitRequestShipping.updateKit(dsmKitId, participantShipment, returnShipment, errorMessage, toAddress, false, ddpInstanceDto); } @@ -392,6 +392,11 @@ public static void getKitStatus() { EasyPostUtil easyPostUtil = new EasyPostUtil(ddpInstance.getName()); SimpleResult results = inTransaction((conn) -> { + DDPInstanceDto ddpInstanceDto = new DDPInstanceDto.Builder() + .withInstanceName(ddpInstance.getName()) + .withEsParticipantIndex(ddpInstance.getParticipantIndexES()) + .build(); + for (KitRequestShipping kitRequest : kitRequestShippingList) { if (StringUtils.isNotBlank(kitRequest.getEasypostToId()) && kitRequest.getEasypostToId().startsWith("shp_")) { try { @@ -430,10 +435,6 @@ else if (status.equals(detail.getStatus())) { kitMessage = message; } } - DDPInstanceDto ddpInstanceDto = new DDPInstanceDto.Builder() - .withInstanceName(ddpInstance.getName()) - .withEsParticipantIndex(ddpInstance.getParticipantIndexES()) - .build(); SimpleResult result = updateKit(conn, status, deliveredDate, kitMessage, kitRequest.getDsmKitId(), ddpInstanceDto); if (result.resultException != null) { @@ -513,7 +514,7 @@ private static SimpleResult updateKit(@NonNull Connection conn, String status, L kitRequestShipping.setMessage(message); kitRequestShipping.setDsmKitId(dsmKitId); - UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, "dsmKitId", "dsmKitId", dsmKitId) + UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, ESObjectConstants.DSM_KIT_ID, ESObjectConstants.DSM_KIT_ID, dsmKitId) .export(); return dbVals; @@ -551,7 +552,7 @@ public static List findSpecialBehaviorKits(@NonNull Notifica boolean specialBehavior = InstanceSettings.shouldKitBehaveDifferently(participant, uploaded); if (specialBehavior) { KitRequestShipping.deactivateKitRequest(kit.getDsmKitRequestId(), SYSTEM_AUTOMATICALLY_DEACTIVATED + ": " + uploaded.getValue(), - DSMServer.getDDPEasypostApiKey(ddpInstance.getName()), "System", + DSMServer.getDDPEasypostApiKey(ddpInstance.getName()), SystemUtil.SYSTEM, new DDPInstanceDao().getDDPInstanceByInstanceName(ddpInstance.getName()).orElseThrow()); if (InstanceSettings.TYPE_NOTIFICATION.equals(uploaded.getType())) { String message = kitType.getName() + " kit for participant " + kit.getParticipantId() + " (" + kit.getCollaboratorParticipantId() From 0907cdb6a6e73e8ea5ef2cd69628405b17c3abef Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 3 Feb 2022 11:35:07 +0400 Subject: [PATCH 377/405] DDP-7443 | remove dead code --- src/main/java/org/broadinstitute/dsm/db/Participant.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/Participant.java b/src/main/java/org/broadinstitute/dsm/db/Participant.java index 367e0831d..2a8524909 100644 --- a/src/main/java/org/broadinstitute/dsm/db/Participant.java +++ b/src/main/java/org/broadinstitute/dsm/db/Participant.java @@ -205,11 +205,6 @@ public static Map getParticipants(@NonNull String realm) { return getParticipants(realm, null); } - public static List getParticipantsByIds(@NonNull String realm, List participantIds) { - String queryAddition = " AND p.ddp_participant_id IN (?)".replace("?", DBUtil.participantIdsInClause(participantIds)); - return new ArrayList<>(getParticipants(realm, queryAddition).values()); - } - public static Map getParticipants(@NonNull String realm, String queryAddition) { logger.info("Collection participant information"); Map participants = new HashMap<>(); From 9be73e09833973cb36ae7a0002a9c54fb5196bab Mon Sep 17 00:00:00 2001 From: gmakhara Date: Thu, 3 Feb 2022 19:15:26 +0400 Subject: [PATCH 378/405] DDP-7443 | change strategy of building query --- ...lderFactory.java => BaseQueryBuilder.java} | 52 +++++++++++-------- .../filter/query/CollectionQueryBuilder.java | 23 ++++++-- .../filter/query/DsmAbstractQueryBuilder.java | 8 +-- .../filter/query/SingleQueryBuilder.java | 15 ++++-- ...oryTest.java => BaseQueryBuilderTest.java} | 8 +-- 5 files changed, 67 insertions(+), 39 deletions(-) rename src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/{QueryBuilderFactory.java => BaseQueryBuilder.java} (74%) rename src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/{QueryBuilderFactoryTest.java => BaseQueryBuilderTest.java} (72%) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryBuilderFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/BaseQueryBuilder.java similarity index 74% rename from src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryBuilderFactory.java rename to src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/BaseQueryBuilder.java index b122d363b..5a9add6ac 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryBuilderFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/BaseQueryBuilder.java @@ -13,12 +13,14 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.RangeQueryBuilder; import org.elasticsearch.index.query.RegexpQueryBuilder; -import org.elasticsearch.index.query.TermQueryBuilder; -import org.elasticsearch.index.query.WildcardQueryBuilder; -public class QueryBuilderFactory { - public static QueryBuilder buildQueryBuilder(Operator operator, QueryPayload payload, - BaseSplitter splitter) { +public abstract class BaseQueryBuilder { + + protected Operator operator; + protected QueryPayload payload; + protected BaseSplitter splitter; + + protected QueryBuilder buildQueryBuilder() { QueryBuilder qb; switch (operator) { case LIKE: @@ -26,33 +28,33 @@ public static QueryBuilder buildQueryBuilder(Operator operator, QueryPayload pay case DATE: case DIAMOND_EQUALS: case STR_DATE: - qb = new MatchQueryBuilder(payload.getFieldName(), payload.getValues()[0]); + qb = build(new MatchQueryBuilder(payload.getFieldName(), payload.getValues()[0])); break; case GREATER_THAN_EQUALS: RangeQueryBuilder greaterRangeQuery = new RangeQueryBuilder(payload.getFieldName()); greaterRangeQuery.gte(payload.getValues()[0]); - qb = greaterRangeQuery; + qb = build(greaterRangeQuery); break; case LESS_THAN_EQUALS: RangeQueryBuilder lessRangeQuery = new RangeQueryBuilder(payload.getFieldName()); lessRangeQuery.lte(payload.getValues()[0]); - qb = lessRangeQuery; + qb = build(lessRangeQuery); break; case DATE_GREATER: RangeQueryBuilder dateGreaterQuery = new RangeQueryBuilder(payload.getFieldName()); dateGreaterQuery.gte(payload.getValues()[0]); - qb = dateGreaterQuery; + qb = build(dateGreaterQuery); break; case DATE_LESS: RangeQueryBuilder dateLessQuery = new RangeQueryBuilder(payload.getFieldName()); dateLessQuery.lte(payload.getValues()[0]); - qb = dateLessQuery; + qb = build(dateLessQuery); break; case IS_NOT_NULL: - qb = buildIsNotNullAndEmpty(payload); + qb = buildIsNotNullAndEmpty(); break; case IS_NULL: - qb = buildIsNullQuery(payload); + qb = buildIsNullQuery(); break; case MULTIPLE_OPTIONS: BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); @@ -60,7 +62,7 @@ public static QueryBuilder buildQueryBuilder(Operator operator, QueryPayload pay for (Object value : values) { boolQueryBuilder.should(new MatchQueryBuilder(payload.getFieldName(), value)); } - qb = boolQueryBuilder; + qb = build(boolQueryBuilder); break; case JSON_EXTRACT: Object[] dynamicFieldValues = payload.getValues(); @@ -78,11 +80,12 @@ public static QueryBuilder buildQueryBuilder(Operator operator, QueryPayload pay } } else { if (jsonExtractSplitter.getDecoratedSplitter() instanceof IsNullSplitter) { - qb = buildIsNullQuery(payload); + qb = buildIsNullQuery(); } else { - qb = buildIsNotNullAndEmpty(payload); + qb = buildIsNotNullAndEmpty(); } } + qb = build(qb); break; default: throw new IllegalArgumentException(Operator.UNKNOWN_OPERATOR); @@ -90,20 +93,23 @@ public static QueryBuilder buildQueryBuilder(Operator operator, QueryPayload pay return qb; } - private static QueryBuilder buildIsNotNullAndEmpty(QueryPayload payload) { + private QueryBuilder buildIsNotNullAndEmpty() { BoolQueryBuilder isNotNullAndNotEmpty = new BoolQueryBuilder(); isNotNullAndNotEmpty.must(new ExistsQueryBuilder(payload.getFieldName())); isNotNullAndNotEmpty.must(new RegexpQueryBuilder(payload.getFieldName(), DsmAbstractQueryBuilder.ONE_OR_MORE_REGEX)); return isNotNullAndNotEmpty; } - private static QueryBuilder buildIsNullQuery(QueryPayload payload) { - BoolQueryBuilder isNullQuery = new BoolQueryBuilder(); + protected abstract QueryBuilder build(QueryBuilder queryBuilder); + + private QueryBuilder buildIsNullQuery() { BoolQueryBuilder existsWithEmpty = new BoolQueryBuilder(); - existsWithEmpty.must(new ExistsQueryBuilder(payload.getFieldName())); - existsWithEmpty.mustNot(new WildcardQueryBuilder(payload.getFieldName(), DsmAbstractQueryBuilder.WILDCARD)); - isNullQuery.should(existsWithEmpty); - isNullQuery.should(new BoolQueryBuilder().mustNot(new ExistsQueryBuilder(payload.getFieldName()))); - return isNullQuery; + existsWithEmpty.mustNot(build(new ExistsQueryBuilder(payload.getFieldName()))); + return existsWithEmpty; } + + protected abstract QueryBuilder buildEachQuery(Operator operator, + QueryPayload queryPayload, + BaseSplitter splitter); + } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/CollectionQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/CollectionQueryBuilder.java index 6b9091422..d081620c1 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/CollectionQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/CollectionQueryBuilder.java @@ -1,16 +1,29 @@ package org.broadinstitute.dsm.model.elastic.filter.query; import org.apache.lucene.search.join.ScoreMode; -import org.broadinstitute.dsm.model.elastic.filter.FilterStrategy; +import org.broadinstitute.dsm.model.elastic.filter.Operator; +import org.broadinstitute.dsm.model.elastic.filter.splitter.BaseSplitter; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.ExistsQueryBuilder; import org.elasticsearch.index.query.NestedQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.WildcardQueryBuilder; -public class CollectionQueryBuilder extends DsmAbstractQueryBuilder { +public class CollectionQueryBuilder extends BaseQueryBuilder { - public CollectionQueryBuilder() {} + @Override + protected QueryBuilder build(QueryBuilder queryBuilder) { + return new NestedQueryBuilder(payload.getPath(), queryBuilder, ScoreMode.Avg); + } @Override - protected void buildEachQuery(FilterStrategy filterStrategy) { - filterStrategy.build(boolQueryBuilder, new NestedQueryBuilder(buildPath(), queryBuilder, ScoreMode.Avg)); + protected QueryBuilder buildEachQuery(Operator operator, + QueryPayload queryPayload, + BaseSplitter splitter) { + this.operator = operator; + this.payload = queryPayload; + this.splitter = splitter; + return buildQueryBuilder(); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/DsmAbstractQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/DsmAbstractQueryBuilder.java index 367fda6ae..a2e952e2f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/DsmAbstractQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/DsmAbstractQueryBuilder.java @@ -28,6 +28,7 @@ public abstract class DsmAbstractQueryBuilder { protected QueryBuilder queryBuilder; protected BaseSplitter splitter; protected AndOrFilterSeparator filterSeparator; + private BaseQueryBuilder baseQueryBuilder; public DsmAbstractQueryBuilder(String filter, Parser parser) { this(); @@ -75,8 +76,9 @@ protected void buildUpQuery(List filterValues, FilterStrategy filterStra splitter = SplitterFactory.createSplitter(operator, filterValue); splitter.setFilter(filterValue); QueryPayload queryPayload = new QueryPayload(buildPath(), splitter.getInnerProperty(), parser.parse(splitter.getValue())); - queryBuilder = QueryBuilderFactory.buildQueryBuilder(operator, queryPayload, splitter); - buildEachQuery(filterStrategy); +// queryBuilder = BaseQueryBuilder.buildQueryBuilder(operator, queryPayload, splitter); + filterStrategy.build(boolQueryBuilder, baseQueryBuilder.buildEachQuery(operator, queryPayload, splitter)); +// buildEachQuery(filterStrategy); } } @@ -84,6 +86,6 @@ protected String buildPath() { return DSM_WITH_DOT + Util.TABLE_ALIAS_MAPPINGS.get(splitter.getAlias()).getPropertyName(); } - protected abstract void buildEachQuery(FilterStrategy filterStrategy); +// protected abstract void buildEachQuery(FilterStrategy filterStrategy); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/SingleQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/SingleQueryBuilder.java index 92a1a8f2e..80ccd6cbe 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/SingleQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/SingleQueryBuilder.java @@ -1,13 +1,20 @@ package org.broadinstitute.dsm.model.elastic.filter.query; import org.broadinstitute.dsm.model.elastic.filter.FilterStrategy; +import org.broadinstitute.dsm.model.elastic.filter.Operator; +import org.broadinstitute.dsm.model.elastic.filter.splitter.BaseSplitter; +import org.elasticsearch.index.query.QueryBuilder; -public class SingleQueryBuilder extends DsmAbstractQueryBuilder { +public class SingleQueryBuilder extends BaseQueryBuilder { - public SingleQueryBuilder() {} @Override - protected void buildEachQuery(FilterStrategy filterStrategy) { - filterStrategy.build(boolQueryBuilder, queryBuilder); + protected QueryBuilder build(QueryBuilder queryBuilder) { + return queryBuilder; + } + + @Override + protected QueryBuilder buildEachQuery(Operator operator, QueryPayload queryPayload, BaseSplitter splitter) { + return null; } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryBuilderFactoryTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/BaseQueryBuilderTest.java similarity index 72% rename from src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryBuilderFactoryTest.java rename to src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/BaseQueryBuilderTest.java index 0cc0641c3..58546870d 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryBuilderFactoryTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/BaseQueryBuilderTest.java @@ -8,20 +8,20 @@ import org.elasticsearch.index.query.RangeQueryBuilder; import org.junit.Test; -public class QueryBuilderFactoryTest { +public class BaseQueryBuilderTest { @Test public void buildQueryBuilder() { QueryPayload payload = new QueryPayload("dsm.medicalRecord", "medicalRecordId", new Integer[] {10}); Operator operator = Operator.EQUALS; - QueryBuilder queryBuilder = QueryBuilderFactory.buildQueryBuilder(operator, payload, null); + QueryBuilder queryBuilder = BaseQueryBuilder.buildQueryBuilder(); assertTrue(queryBuilder instanceof MatchQueryBuilder); operator = Operator.LIKE; - queryBuilder = QueryBuilderFactory.buildQueryBuilder(operator, payload, null); + queryBuilder = BaseQueryBuilder.buildQueryBuilder(); assertTrue(queryBuilder instanceof MatchQueryBuilder); operator = Operator.GREATER_THAN_EQUALS; - queryBuilder = QueryBuilderFactory.buildQueryBuilder(operator, payload, null); + queryBuilder = BaseQueryBuilder.buildQueryBuilder(); assertTrue(queryBuilder instanceof RangeQueryBuilder); } From 2d898c3167f9ffc529e769cf70bc119fe0cef35e Mon Sep 17 00:00:00 2001 From: Nika Date: Fri, 4 Feb 2022 10:12:21 +0400 Subject: [PATCH 379/405] DDP-7443 | refactor code and fix bugs in order to enable "isEmpty" and "isNotEmpty" search capabilities, make tests pass --- .../org/broadinstitute/dsm/model/Filter.java | 3 +- .../dsm/model/elastic/filter/Operator.java | 9 ++--- .../filter/query/BaseQueryBuilder.java | 38 ++++++------------ .../filter/query/CollectionQueryBuilder.java | 11 ------ .../filter/query/DsmAbstractQueryBuilder.java | 27 ++++--------- .../filter/query/SingleQueryBuilder.java | 9 ----- .../filter/splitter/SplitterFactory.java | 4 +- .../model/elastic/filter/OperatorTest.java | 4 +- .../filter/query/BaseQueryBuilderTest.java | 30 +++++++++----- .../query/CollectionQueryBuilderTest.java | 39 +++++++++---------- .../filter/query/SingleQueryBuilderTest.java | 11 +++--- .../splitter/DateGreaterSplitterTest.java | 4 +- .../splitter/DateLowerSplitterTest.java | 4 +- 13 files changed, 75 insertions(+), 118 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/Filter.java b/src/main/java/org/broadinstitute/dsm/model/Filter.java index ab4de3ae3..7492af76e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/Filter.java +++ b/src/main/java/org/broadinstitute/dsm/model/Filter.java @@ -191,7 +191,6 @@ else if (filter.isNotEmpty()) { } else { filter = convertFilterDateValues(filter); - String notNullQuery = AND + filter.getColumnName(dbElement) + IS_NOT_NULL; String query1 = ""; if (filter.getFilter1() != null && filter.getFilter1().getValue() != null && StringUtils.isNotBlank(String.valueOf(filter.getFilter1().getValue()))) { query1 = generateDateComparisonSql(filter,dbElement, LARGER_EQUALS, filter.getFilter1().getValue(), false); @@ -200,7 +199,7 @@ else if (filter.isNotEmpty()) { if (filter.getFilter2() != null && filter.getFilter2() != null && filter.getFilter2().getValue() != null && StringUtils.isNotBlank(String.valueOf(filter.getFilter2().getValue()))) { query2 = generateDateComparisonSql(filter,dbElement, SMALLER_EQUALS,filter.getFilter2().getValue(), true); } - finalQuery = query1 + query2 + notNullQuery; + finalQuery = query1 + query2; } } else if (ADDITIONAL_VALUES.equals(filter.getType())) { diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java index ea172ba5d..9abae8dba 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java @@ -1,7 +1,6 @@ package org.broadinstitute.dsm.model.elastic.filter; import java.util.*; -import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; @@ -18,8 +17,8 @@ public enum Operator { DIAMOND_EQUALS(Filter.DIAMOND_EQUALS), MULTIPLE_OPTIONS(Operator.MULTIPLE_OPTIONS_INDICATOR), STR_DATE(Filter.DATE_FORMAT), - DATE_GREATER(Filter.DATE_GREATER), - DATE_LESS(Filter.DATE_LESS), + DATE_GREATER_THAN_EQUALS(Filter.DATE_GREATER), + DATE_LESS_THAN_EQUALS(Filter.DATE_LESS), JSON_EXTRACT(Filter.JSON_EXTRACT), DATE(Filter.DATE); @@ -57,9 +56,9 @@ public static Operator extract(String filter) { case "STR_TO_DATE =": return Operator.STR_DATE; case "STR_TO_DATE <=": - return Operator.DATE_LESS; + return Operator.DATE_LESS_THAN_EQUALS; case "STR_TO_DATE >=": - return Operator.DATE_GREATER; + return Operator.DATE_GREATER_THAN_EQUALS; case "DATE =": return Operator.DATE; case "NOT <=>": diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/BaseQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/BaseQueryBuilder.java index 5a9add6ac..54f9cc4df 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/BaseQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/BaseQueryBuilder.java @@ -2,17 +2,8 @@ import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.elastic.filter.Operator; -import org.broadinstitute.dsm.model.elastic.filter.splitter.BaseSplitter; -import org.broadinstitute.dsm.model.elastic.filter.splitter.GreaterThanEqualsSplitter; -import org.broadinstitute.dsm.model.elastic.filter.splitter.IsNullSplitter; -import org.broadinstitute.dsm.model.elastic.filter.splitter.JsonExtractSplitter; -import org.broadinstitute.dsm.model.elastic.filter.splitter.LessThanEqualsSplitter; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.ExistsQueryBuilder; -import org.elasticsearch.index.query.MatchQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.RangeQueryBuilder; -import org.elasticsearch.index.query.RegexpQueryBuilder; +import org.broadinstitute.dsm.model.elastic.filter.splitter.*; +import org.elasticsearch.index.query.*; public abstract class BaseQueryBuilder { @@ -31,25 +22,17 @@ protected QueryBuilder buildQueryBuilder() { qb = build(new MatchQueryBuilder(payload.getFieldName(), payload.getValues()[0])); break; case GREATER_THAN_EQUALS: + case DATE_GREATER_THAN_EQUALS: RangeQueryBuilder greaterRangeQuery = new RangeQueryBuilder(payload.getFieldName()); greaterRangeQuery.gte(payload.getValues()[0]); qb = build(greaterRangeQuery); break; case LESS_THAN_EQUALS: + case DATE_LESS_THAN_EQUALS: RangeQueryBuilder lessRangeQuery = new RangeQueryBuilder(payload.getFieldName()); lessRangeQuery.lte(payload.getValues()[0]); qb = build(lessRangeQuery); break; - case DATE_GREATER: - RangeQueryBuilder dateGreaterQuery = new RangeQueryBuilder(payload.getFieldName()); - dateGreaterQuery.gte(payload.getValues()[0]); - qb = build(dateGreaterQuery); - break; - case DATE_LESS: - RangeQueryBuilder dateLessQuery = new RangeQueryBuilder(payload.getFieldName()); - dateLessQuery.lte(payload.getValues()[0]); - qb = build(dateLessQuery); - break; case IS_NOT_NULL: qb = buildIsNotNullAndEmpty(); break; @@ -67,7 +50,6 @@ protected QueryBuilder buildQueryBuilder() { case JSON_EXTRACT: Object[] dynamicFieldValues = payload.getValues(); JsonExtractSplitter jsonExtractSplitter = (JsonExtractSplitter) splitter; -// buildQueryBuilder(jsonExtractSplitter.getOperator()); if (!StringUtils.EMPTY.equals(dynamicFieldValues[0])) { if (jsonExtractSplitter.getDecoratedSplitter() instanceof GreaterThanEqualsSplitter) { qb = new RangeQueryBuilder(payload.getFieldName()); @@ -95,8 +77,7 @@ protected QueryBuilder buildQueryBuilder() { private QueryBuilder buildIsNotNullAndEmpty() { BoolQueryBuilder isNotNullAndNotEmpty = new BoolQueryBuilder(); - isNotNullAndNotEmpty.must(new ExistsQueryBuilder(payload.getFieldName())); - isNotNullAndNotEmpty.must(new RegexpQueryBuilder(payload.getFieldName(), DsmAbstractQueryBuilder.ONE_OR_MORE_REGEX)); + isNotNullAndNotEmpty.must(build(new ExistsQueryBuilder(payload.getFieldName()))); return isNotNullAndNotEmpty; } @@ -108,8 +89,13 @@ private QueryBuilder buildIsNullQuery() { return existsWithEmpty; } - protected abstract QueryBuilder buildEachQuery(Operator operator, + public QueryBuilder buildEachQuery(Operator operator, QueryPayload queryPayload, - BaseSplitter splitter); + BaseSplitter splitter) { + this.operator = operator; + this.payload = queryPayload; + this.splitter = splitter; + return buildQueryBuilder(); + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/CollectionQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/CollectionQueryBuilder.java index d081620c1..0390fd0f3 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/CollectionQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/CollectionQueryBuilder.java @@ -15,16 +15,5 @@ public class CollectionQueryBuilder extends BaseQueryBuilder { protected QueryBuilder build(QueryBuilder queryBuilder) { return new NestedQueryBuilder(payload.getPath(), queryBuilder, ScoreMode.Avg); } - - @Override - protected QueryBuilder buildEachQuery(Operator operator, - QueryPayload queryPayload, - BaseSplitter splitter) { - this.operator = operator; - this.payload = queryPayload; - this.splitter = splitter; - return buildQueryBuilder(); - } - } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/DsmAbstractQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/DsmAbstractQueryBuilder.java index a2e952e2f..46f87cc6c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/DsmAbstractQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/DsmAbstractQueryBuilder.java @@ -1,8 +1,5 @@ package org.broadinstitute.dsm.model.elastic.filter.query; -import java.util.List; -import java.util.Map; - import org.broadinstitute.dsm.model.Filter; import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.export.parse.Parser; @@ -17,11 +14,12 @@ import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; -public abstract class DsmAbstractQueryBuilder { +import java.util.List; +import java.util.Map; + +public class DsmAbstractQueryBuilder { - public static final String ONE_OR_MORE_REGEX = ".+"; protected static final String DSM_WITH_DOT = ESObjectConstants.DSM + DBConstants.ALIAS_DELIMITER; - public static final String WILDCARD = "*"; protected String filter; protected Parser parser; protected BoolQueryBuilder boolQueryBuilder; @@ -30,21 +28,15 @@ public abstract class DsmAbstractQueryBuilder { protected AndOrFilterSeparator filterSeparator; private BaseQueryBuilder baseQueryBuilder; - public DsmAbstractQueryBuilder(String filter, Parser parser) { - this(); - this.parser = parser; - this.filter = filter; - this.filterSeparator = new AndOrFilterSeparator(this.filter); - } - public static DsmAbstractQueryBuilder of(String alias) { boolean isCollection = Util.TABLE_ALIAS_MAPPINGS.get(alias).isCollection(); return isCollection - ? new CollectionQueryBuilder() - : new SingleQueryBuilder(); + ? new DsmAbstractQueryBuilder(new CollectionQueryBuilder()) + : new DsmAbstractQueryBuilder(new SingleQueryBuilder()); } - public DsmAbstractQueryBuilder() { + private DsmAbstractQueryBuilder(BaseQueryBuilder baseQueryBuilder) { + this.baseQueryBuilder = baseQueryBuilder; boolQueryBuilder = new BoolQueryBuilder(); } @@ -76,9 +68,7 @@ protected void buildUpQuery(List filterValues, FilterStrategy filterStra splitter = SplitterFactory.createSplitter(operator, filterValue); splitter.setFilter(filterValue); QueryPayload queryPayload = new QueryPayload(buildPath(), splitter.getInnerProperty(), parser.parse(splitter.getValue())); -// queryBuilder = BaseQueryBuilder.buildQueryBuilder(operator, queryPayload, splitter); filterStrategy.build(boolQueryBuilder, baseQueryBuilder.buildEachQuery(operator, queryPayload, splitter)); -// buildEachQuery(filterStrategy); } } @@ -86,6 +76,5 @@ protected String buildPath() { return DSM_WITH_DOT + Util.TABLE_ALIAS_MAPPINGS.get(splitter.getAlias()).getPropertyName(); } -// protected abstract void buildEachQuery(FilterStrategy filterStrategy); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/SingleQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/SingleQueryBuilder.java index 80ccd6cbe..5bcf9cc1f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/SingleQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/SingleQueryBuilder.java @@ -1,20 +1,11 @@ package org.broadinstitute.dsm.model.elastic.filter.query; -import org.broadinstitute.dsm.model.elastic.filter.FilterStrategy; -import org.broadinstitute.dsm.model.elastic.filter.Operator; -import org.broadinstitute.dsm.model.elastic.filter.splitter.BaseSplitter; import org.elasticsearch.index.query.QueryBuilder; public class SingleQueryBuilder extends BaseQueryBuilder { - @Override protected QueryBuilder build(QueryBuilder queryBuilder) { return queryBuilder; } - - @Override - protected QueryBuilder buildEachQuery(Operator operator, QueryPayload queryPayload, BaseSplitter splitter) { - return null; - } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/SplitterFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/SplitterFactory.java index 86cfeda90..3f749e0eb 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/SplitterFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/SplitterFactory.java @@ -33,10 +33,10 @@ public static BaseSplitter createSplitter(Operator operator, String filterValue) case DATE: splitter = new DateSplitter(); break; - case DATE_GREATER: + case DATE_GREATER_THAN_EQUALS: splitter = new DateGreaterSplitter(); break; - case DATE_LESS: + case DATE_LESS_THAN_EQUALS: splitter = new DateLowerSplitter(); break; case IS_NOT_NULL: diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java index 922fa6102..6efcf04c8 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/OperatorTest.java @@ -112,14 +112,14 @@ public void extractStrToDateOperator() { public void extractGreaterStrToDateOperator() { String filter = "m.mr_received >= STR_TO_DATE('1964-01-14','%Y-%m-%d')"; Operator operator = Operator.extract(filter); - assertEquals(Operator.DATE_GREATER, operator); + assertEquals(Operator.DATE_GREATER_THAN_EQUALS, operator); } @Test public void extractLessStrToDateOperator() { String filter = "m.mr_received <= STR_TO_DATE('1964-01-14','%Y-%m-%d')"; Operator operator = Operator.extract(filter); - assertEquals(Operator.DATE_LESS, operator); + assertEquals(Operator.DATE_LESS_THAN_EQUALS, operator); } @Test diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/BaseQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/BaseQueryBuilderTest.java index 58546870d..4aef40934 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/BaseQueryBuilderTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/BaseQueryBuilderTest.java @@ -1,13 +1,13 @@ package org.broadinstitute.dsm.model.elastic.filter.query; -import static org.junit.Assert.*; - import org.broadinstitute.dsm.model.elastic.filter.Operator; import org.elasticsearch.index.query.MatchQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.NestedQueryBuilder; import org.elasticsearch.index.query.RangeQueryBuilder; import org.junit.Test; +import static org.junit.Assert.assertTrue; + public class BaseQueryBuilderTest { @@ -15,14 +15,26 @@ public class BaseQueryBuilderTest { public void buildQueryBuilder() { QueryPayload payload = new QueryPayload("dsm.medicalRecord", "medicalRecordId", new Integer[] {10}); Operator operator = Operator.EQUALS; - QueryBuilder queryBuilder = BaseQueryBuilder.buildQueryBuilder(); - assertTrue(queryBuilder instanceof MatchQueryBuilder); + + CollectionQueryBuilder collectionQueryBuilder = new CollectionQueryBuilder(); + collectionQueryBuilder.operator = operator; + collectionQueryBuilder.payload = payload; + + NestedQueryBuilder queryBuilder = (NestedQueryBuilder) collectionQueryBuilder.buildQueryBuilder(); + assertTrue(queryBuilder.query() instanceof MatchQueryBuilder); + operator = Operator.LIKE; - queryBuilder = BaseQueryBuilder.buildQueryBuilder(); - assertTrue(queryBuilder instanceof MatchQueryBuilder); + collectionQueryBuilder.operator = operator; + + queryBuilder = (NestedQueryBuilder) collectionQueryBuilder.buildQueryBuilder(); + assertTrue(queryBuilder.query() instanceof MatchQueryBuilder); + + operator = Operator.GREATER_THAN_EQUALS; - queryBuilder = BaseQueryBuilder.buildQueryBuilder(); - assertTrue(queryBuilder instanceof RangeQueryBuilder); + collectionQueryBuilder.operator = operator; + queryBuilder = (NestedQueryBuilder) collectionQueryBuilder.buildQueryBuilder(); + + assertTrue(queryBuilder.query() instanceof RangeQueryBuilder); } } \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/CollectionQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/CollectionQueryBuilderTest.java index e6852dc9b..70f7c65b1 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/CollectionQueryBuilderTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/CollectionQueryBuilderTest.java @@ -4,7 +4,6 @@ import org.apache.lucene.search.join.ScoreMode; import org.broadinstitute.dsm.model.elastic.filter.FilterParser; -import org.broadinstitute.dsm.model.elastic.filter.query.CollectionQueryBuilder; import org.elasticsearch.index.query.*; import org.junit.Assert; import org.junit.Before; @@ -12,12 +11,12 @@ public class CollectionQueryBuilderTest { - CollectionQueryBuilder collectionQueryBuilder; + DsmAbstractQueryBuilder dsmAbstractQueryBuilder; @Before public void setUp() { - collectionQueryBuilder = new CollectionQueryBuilder(); - collectionQueryBuilder.setParser(new FilterParser()); + dsmAbstractQueryBuilder = DsmAbstractQueryBuilder.of("m"); + dsmAbstractQueryBuilder.setParser(new FilterParser()); } @Test @@ -25,8 +24,8 @@ public void collectionBuild() { String filter = "AND m.medicalRecordId = '15' AND m.type = 'PHYSICIAN' OR k.bspCollaboratorSampleId = 'ASCProject_PZ8GJC_SALIVA'"; - collectionQueryBuilder.setFilter(filter); - AbstractQueryBuilder actual = collectionQueryBuilder.build(); + dsmAbstractQueryBuilder.setFilter(filter); + AbstractQueryBuilder actual = dsmAbstractQueryBuilder.build(); AbstractQueryBuilder expected = new BoolQueryBuilder().must(new NestedQueryBuilder("dsm.medicalRecord", new MatchQueryBuilder("dsm.medicalRecord.medicalRecordId", "15"), ScoreMode.Avg)) .must(new NestedQueryBuilder("dsm.medicalRecord", new MatchQueryBuilder("dsm.medicalRecord.type", "PHYSICIAN"), ScoreMode.Avg)) @@ -40,19 +39,18 @@ public void collectionBuild2() { String filter = "AND m.medicalRecordId >= '15' AND m.type LIKE 'PHYSICIAN' OR k.bspCollaboratorSampleId = 'ASCProject_PZ8GJC_SALIVA' AND t.returnDate <= '2015-01-01' AND p.participantId IS NOT NULL"; - collectionQueryBuilder.setFilter(filter); - AbstractQueryBuilder actual = collectionQueryBuilder.build(); + dsmAbstractQueryBuilder.setFilter(filter); + AbstractQueryBuilder actual = dsmAbstractQueryBuilder.build(); BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); boolQueryBuilder.must(new ExistsQueryBuilder("dsm.participant.participantId")); - boolQueryBuilder.must(new RegexpQueryBuilder("dsm.participant.participantId", ".+")); AbstractQueryBuilder expected = new BoolQueryBuilder().must(new NestedQueryBuilder("dsm.medicalRecord", new RangeQueryBuilder("dsm.medicalRecord.medicalRecordId").gte("15"), ScoreMode.Avg)) .must(new NestedQueryBuilder("dsm.medicalRecord", new MatchQueryBuilder("dsm.medicalRecord.type", "PHYSICIAN"), ScoreMode.Avg)) .should(new NestedQueryBuilder("dsm.kitRequestShipping", new MatchQueryBuilder("dsm.kitRequestShipping.bspCollaboratorSampleId", "ASCProject_PZ8GJC_SALIVA"), ScoreMode.Avg)) .must(new NestedQueryBuilder("dsm.tissue", new RangeQueryBuilder("dsm.tissue.returnDate").lte("2015-01-01"), ScoreMode.Avg)) - .must(new NestedQueryBuilder("dsm.participant", boolQueryBuilder, ScoreMode.Avg)); + .must(boolQueryBuilder); Assert.assertEquals(expected, actual); } @@ -62,8 +60,8 @@ public void collectionBuildAgeRange() { String filter = "AND m.age >= '15' AND m.age <= '30'"; - collectionQueryBuilder.setFilter(filter); - AbstractQueryBuilder actual = collectionQueryBuilder.build(); + dsmAbstractQueryBuilder.setFilter(filter); + AbstractQueryBuilder actual = dsmAbstractQueryBuilder.build(); AbstractQueryBuilder expected = new BoolQueryBuilder().must(new NestedQueryBuilder("dsm.medicalRecord", new RangeQueryBuilder("dsm.medicalRecord.age").gte("15"), ScoreMode.Avg)) @@ -78,8 +76,8 @@ public void collectionBuildBoolean() { String filter = "AND m.followUp LIKE '1'"; - collectionQueryBuilder.setFilter(filter); - AbstractQueryBuilder actual = collectionQueryBuilder.build(); + dsmAbstractQueryBuilder.setFilter(filter); + AbstractQueryBuilder actual = dsmAbstractQueryBuilder.build(); AbstractQueryBuilder expected = new BoolQueryBuilder().must(new NestedQueryBuilder("dsm.medicalRecord", new MatchQueryBuilder("dsm.medicalRecord.followUp", true), ScoreMode.Avg)); @@ -92,8 +90,8 @@ public void multipleOptionsQueryBuilder() { String filter = "AND ( oD.request = 'review' OR oD.request = 'no' OR oD.request = 'hold' OR oD.request = 'request' OR oD.request = 'unable To Obtain' OR oD.request = 'sent' OR oD.request = 'received' OR oD.request = 'returned' )"; - collectionQueryBuilder.setFilter(filter); - AbstractQueryBuilder actual = collectionQueryBuilder.build(); + dsmAbstractQueryBuilder.setFilter(filter); + AbstractQueryBuilder actual = dsmAbstractQueryBuilder.build(); BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); boolQueryBuilder.should(new MatchQueryBuilder("dsm.oncHistoryDetail.request", "review")); @@ -116,8 +114,8 @@ public void dateGreaterBuild() { String filter = "AND m.received >= STR_TO_DATE('2012-01-01', %yyyy-%MM-%dd) AND m.received <= STR_TO_DATE('2015-01-01', %yyyy-%MM-%dd)"; - collectionQueryBuilder.setFilter(filter); - AbstractQueryBuilder actual = collectionQueryBuilder.build(); + dsmAbstractQueryBuilder.setFilter(filter); + AbstractQueryBuilder actual = dsmAbstractQueryBuilder.build(); AbstractQueryBuilder expected = new BoolQueryBuilder().must(new NestedQueryBuilder("dsm.medicalRecord", new RangeQueryBuilder("dsm.medicalRecord.received").gte("2012-01-01"), ScoreMode.Avg)) @@ -132,13 +130,12 @@ public void dynamicFieldsQueryBuild() { String filter = "AND JSON_EXTRACT ( m.additional_values_json , '$.seeingIfBugExists' ) = 'true' AND JSON_EXTRACT ( m.additional_values_json , '$.tryAgain' ) IS NOT NULL"; - collectionQueryBuilder.setFilter(filter); - AbstractQueryBuilder actual = collectionQueryBuilder.build(); + dsmAbstractQueryBuilder.setFilter(filter); + AbstractQueryBuilder actual = dsmAbstractQueryBuilder.build(); BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); boolQueryBuilder.must(new ExistsQueryBuilder("dsm.medicalRecord.dynamicFields.tryAgain")); - boolQueryBuilder.must(new RegexpQueryBuilder("dsm.medicalRecord.dynamicFields.tryAgain", ".+")); AbstractQueryBuilder expected = new BoolQueryBuilder().must(new NestedQueryBuilder("dsm.medicalRecord", new MatchQueryBuilder("dsm.medicalRecord.dynamicFields.seeingIfBugExists", true), ScoreMode.Avg)) diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/SingleQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/SingleQueryBuilderTest.java index 9008240d7..6936def9e 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/SingleQueryBuilderTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/SingleQueryBuilderTest.java @@ -1,7 +1,6 @@ package org.broadinstitute.dsm.model.elastic.filter.query; import org.broadinstitute.dsm.model.elastic.filter.FilterParser; -import org.broadinstitute.dsm.model.elastic.filter.query.SingleQueryBuilder; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.MatchQueryBuilder; @@ -11,21 +10,21 @@ public class SingleQueryBuilderTest { - SingleQueryBuilder singleQueryBuilder; + DsmAbstractQueryBuilder dsmAbstractQueryBuilder; @Before public void setUp() { - singleQueryBuilder = new SingleQueryBuilder(); - singleQueryBuilder.setParser(new FilterParser()); + dsmAbstractQueryBuilder = DsmAbstractQueryBuilder.of("p"); + dsmAbstractQueryBuilder.setParser(new FilterParser()); } @Test public void buildEachQuery() { String filter = "AND p.participantId = '1234'"; - singleQueryBuilder.setFilter(filter); + dsmAbstractQueryBuilder.setFilter(filter); - AbstractQueryBuilder actual = singleQueryBuilder.build(); + AbstractQueryBuilder actual = dsmAbstractQueryBuilder.build(); BoolQueryBuilder expected = new BoolQueryBuilder().must(new MatchQueryBuilder("dsm.participant.participantId", "1234")); Assert.assertEquals(expected, actual); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateGreaterSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateGreaterSplitterTest.java index 6d395c56b..57846342c 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateGreaterSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateGreaterSplitterTest.java @@ -1,8 +1,6 @@ package org.broadinstitute.dsm.model.elastic.filter.splitter; import org.broadinstitute.dsm.model.elastic.filter.Operator; -import org.broadinstitute.dsm.model.elastic.filter.splitter.BaseSplitter; -import org.broadinstitute.dsm.model.elastic.filter.splitter.SplitterFactory; import org.junit.Assert; import org.junit.Test; @@ -10,7 +8,7 @@ public class DateGreaterSplitterTest { @Test public void getValue() { - BaseSplitter splitter = SplitterFactory.createSplitter(Operator.DATE_GREATER, ""); + BaseSplitter splitter = SplitterFactory.createSplitter(Operator.DATE_GREATER_THAN_EQUALS, ""); String filter = "AND m.mr_received >= STR_TO_DATE('2000-01-01','%Y-%m-%d')"; splitter.setFilter(filter); Assert.assertEquals("'2000-01-01'", splitter.getValue()[0]); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateLowerSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateLowerSplitterTest.java index 52fda8163..9a72a3980 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateLowerSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/DateLowerSplitterTest.java @@ -1,8 +1,6 @@ package org.broadinstitute.dsm.model.elastic.filter.splitter; import org.broadinstitute.dsm.model.elastic.filter.Operator; -import org.broadinstitute.dsm.model.elastic.filter.splitter.BaseSplitter; -import org.broadinstitute.dsm.model.elastic.filter.splitter.SplitterFactory; import org.junit.Assert; import org.junit.Test; @@ -10,7 +8,7 @@ public class DateLowerSplitterTest { @Test public void getValue() { - BaseSplitter splitter = SplitterFactory.createSplitter(Operator.DATE_LESS, ""); + BaseSplitter splitter = SplitterFactory.createSplitter(Operator.DATE_LESS_THAN_EQUALS, ""); String filter = "AND m.mr_received <= STR_TO_DATE('2000-01-01','%Y-%m-%d')"; splitter.setFilter(filter); Assert.assertEquals("'2000-01-01'", splitter.getValue()[0]); From db2e8e9447ff471ab5d084b4081ca16ca5fcae23 Mon Sep 17 00:00:00 2001 From: Nika Date: Fri, 4 Feb 2022 15:01:44 +0400 Subject: [PATCH 380/405] DDP-7443 | make sure that field names are the same for both client/backend side --- .../dsm/db/KitRequestShipping.java | 24 ++++++++----------- .../dsm/route/DashboardRoute.java | 16 ++++++------- .../dsm/route/KitExpressRoute.java | 4 ++-- .../dsm/route/ParticipantExitRoute.java | 2 +- .../org/broadinstitute/dsm/util/PDFAudit.java | 4 ++-- 5 files changed, 23 insertions(+), 27 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java index b6222701f..afcd927cb 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java @@ -7,7 +7,6 @@ import lombok.Data; import lombok.NonNull; import org.apache.commons.lang3.StringUtils; -import org.apache.lucene.search.join.ScoreMode; import org.broadinstitute.ddp.db.SimpleResult; import org.broadinstitute.ddp.db.TransactionWrapper; import org.broadinstitute.dsm.DSMServer; @@ -22,7 +21,6 @@ import org.broadinstitute.dsm.model.ddp.DDPParticipant; import org.broadinstitute.dsm.model.ddp.KitDetail; import org.broadinstitute.dsm.model.elastic.export.Exportable; -import org.broadinstitute.dsm.model.elastic.export.generate.Generator; import org.broadinstitute.dsm.model.elastic.export.painless.*; import org.broadinstitute.dsm.statics.ApplicationConfigConstants; import org.broadinstitute.dsm.statics.DBConstants; @@ -33,8 +31,6 @@ import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.broadinstitute.dsm.util.KitUtil; import org.eclipse.jetty.util.StringUtil; -import org.elasticsearch.index.query.MatchQueryBuilder; -import org.elasticsearch.index.query.NestedQueryBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -184,7 +180,7 @@ public class KitRequestShipping extends KitRequest { private String realm; @ColumnName (DBConstants.KIT_TYPE_NAME) - private String kitTypeName; + private String kitType; @ColumnName (DBConstants.DEACTIVATION_REASON) private String deactivationReason; @@ -193,7 +189,7 @@ public class KitRequestShipping extends KitRequest { private String kitLabel; @ColumnName (DBConstants.KIT_TEST_RESULT) - private String testResult; + private String result; @ColumnName (DBConstants.DSM_SCAN_DATE) @DbDateConversion(SqlDateConverter.EPOCH) @@ -250,13 +246,13 @@ public class KitRequestShipping extends KitRequest { public KitRequestShipping() {} - public KitRequestShipping(String collaboratorParticipantId, String kitTypeName, Long dsmKitRequestId, Long scanDate, Boolean error, - Long receiveDate, Long deactivatedDate, String testResult, + public KitRequestShipping(String collaboratorParticipantId, String kitType, Long dsmKitRequestId, Long scanDate, Boolean error, + Long receiveDate, Long deactivatedDate, String result, String upsTrackingStatus, String upsReturnStatus, String externalOrderStatus, String externalOrderNumber, Long externalOrderDate, Boolean careEvolve, String uploadReason) { - this(null, collaboratorParticipantId, null, null, null, kitTypeName, dsmKitRequestId, null, null, null, + this(null, collaboratorParticipantId, null, null, null, kitType, dsmKitRequestId, null, null, null, null, null, null, null, scanDate, error, null, receiveDate, null, deactivatedDate, null, null, null, null, null, null, externalOrderNumber, null, externalOrderStatus, null, - testResult, + result, upsTrackingStatus, upsReturnStatus, externalOrderDate, careEvolve, uploadReason, null, null, null); } @@ -279,19 +275,19 @@ public KitRequestShipping(Long dsmKitRequestId, Long dsmKitId, String easypostTo // shippingId = ddp_label !!! public KitRequestShipping(String participantId, String collaboratorParticipantId, String bspCollaboratorSampleId, String shippingId, String realm, - String kitTypeName, Long dsmKitRequestId, Long dsmKitId, String labelUrlTo, String labelUrlReturn, + String kitType, Long dsmKitRequestId, Long dsmKitId, String labelUrlTo, String labelUrlReturn, String trackingNumberTo, String trackingReturnId, String easypostTrackingToUrl, String trackingUrlReturn, Long scanDate, Boolean error, String message, Long receiveDate, String easypostAddressId, Long deactivatedDate, String deactivationReason, String kitLabel, Boolean express, String easypostToId, Long labelDate, String easypostShipmentStatus, - String externalOrderNumber, Boolean noReturn, String externalOrderStatus, String createdBy, String testResult, + String externalOrderNumber, Boolean noReturn, String externalOrderStatus, String createdBy, String result, String upsTrackingStatus, String upsReturnStatus, Long externalOrderDate, Boolean careEvolve, String uploadReason, String receiveDateString, String hruid, String gender) { super(dsmKitRequestId, participantId, null, shippingId, externalOrderNumber, null, externalOrderStatus, null, externalOrderDate); this.collaboratorParticipantId = collaboratorParticipantId; this.bspCollaboratorSampleId = bspCollaboratorSampleId; this.realm = realm; - this.kitTypeName = kitTypeName; + this.kitType = kitType; this.dsmKitId = dsmKitId; this.labelUrlTo = labelUrlTo; this.labelUrlReturn = labelUrlReturn; @@ -313,7 +309,7 @@ public KitRequestShipping(String participantId, String collaboratorParticipantId this.easypostShipmentStatus = easypostShipmentStatus; this.noReturn = noReturn; this.createdBy = createdBy; - this.testResult = testResult; + this.result = result; this.upsTrackingStatus = upsTrackingStatus; this.upsReturnStatus = upsReturnStatus; this.careEvolve = careEvolve; diff --git a/src/main/java/org/broadinstitute/dsm/route/DashboardRoute.java b/src/main/java/org/broadinstitute/dsm/route/DashboardRoute.java index 3f7a3ab42..3e09a3e7e 100644 --- a/src/main/java/org/broadinstitute/dsm/route/DashboardRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/DashboardRoute.java @@ -499,22 +499,22 @@ private void countKits(@NonNull List kits, @NonNull Set kitTypes = org.broadinstitute.dsm.model.KitType.getKitLookup(); - String key = kitRequest.getKitTypeName() + "_" + ddpInstanceDto.getDdpInstanceId(); + String key = kitRequest.getKitType() + "_" + ddpInstanceDto.getDdpInstanceId(); KitType kitType = kitTypes.get(key); Map carrierServiceTypes = @@ -143,7 +143,7 @@ private EasypostLabelRate getRateForOvernightExpress(@NonNull String kitRequestI DDPInstance ddpInstance = DDPInstance.getDDPInstance(kitRequest.getRealm()); HashMap kitTypes = org.broadinstitute.dsm.model.KitType.getKitLookup(); - String key = kitRequest.getKitTypeName() + "_" + ddpInstance.getDdpInstanceId(); + String key = kitRequest.getKitType() + "_" + ddpInstance.getDdpInstanceId(); KitType kitType = kitTypes.get(key); Map carrierServiceTypes = KitRequestSettings.getKitRequestSettings(ddpInstance.getDdpInstanceId()); diff --git a/src/main/java/org/broadinstitute/dsm/route/ParticipantExitRoute.java b/src/main/java/org/broadinstitute/dsm/route/ParticipantExitRoute.java index 6ec98040b..284d6e8cf 100644 --- a/src/main/java/org/broadinstitute/dsm/route/ParticipantExitRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/ParticipantExitRoute.java @@ -65,7 +65,7 @@ public Object processRequest(Request request, Response response, String userId) for (KitRequestShipping kit : kitRequests) { if (kit.getScanDate() != 0 && kit.getReceiveDate() == 0) { String discardId = KitDiscard.addKitToDiscard(kit.getDsmKitRequestId(), KitDiscard.HOLD); - kitsNeedAction.add(new KitDiscard(discardId, kit.getKitTypeName(), KitDiscard.HOLD)); + kitsNeedAction.add(new KitDiscard(discardId, kit.getKitType(), KitDiscard.HOLD)); } else { //refund label of kits which are not sent yet diff --git a/src/main/java/org/broadinstitute/dsm/util/PDFAudit.java b/src/main/java/org/broadinstitute/dsm/util/PDFAudit.java index 70ad828e1..a7826dff6 100644 --- a/src/main/java/org/broadinstitute/dsm/util/PDFAudit.java +++ b/src/main/java/org/broadinstitute/dsm/util/PDFAudit.java @@ -26,14 +26,14 @@ public void checkAndSavePDF() { // ddp with 'standard' consent and release pdfs DDPInstance instance = DDPInstance.getDDPInstanceWithRole(request.getRealm(), DBConstants.PDF_DOWNLOAD_CONSENT); //the role for release will get checked in makePDF if (instance != null && StringUtils.isNotBlank(instance.getBaseUrl()) && instance.isHasRole()) { - DDPRequestUtil.makeStandardPDF(instance, request.getParticipantId(), request.getCreatedBy(), request.getKitTypeName()); + DDPRequestUtil.makeStandardPDF(instance, request.getParticipantId(), request.getCreatedBy(), request.getKitType()); } else { // ddp without 'standard' consent and release pdfs instance = DDPInstance.getDDPInstanceWithRole(request.getRealm(), DBConstants.PDF_DOWNLOAD); //the role for release will get checked in makePDF if (instance != null && StringUtils.isNotBlank(instance.getBaseUrl()) && StringUtils.isNotBlank(instance.getParticipantIndexES()) && instance.isHasRole()) { - DDPRequestUtil.makeNonStandardPDF(instance, request.getParticipantId(), request.getCreatedBy(), request.getKitTypeName()); + DDPRequestUtil.makeNonStandardPDF(instance, request.getParticipantId(), request.getCreatedBy(), request.getKitType()); } } newAuditKit = Math.max(newAuditKit, request.getDsmKitRequestId()); From 20658093c91b7722fa4c3c715e8922d71967ff51 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 4 Feb 2022 15:32:40 +0400 Subject: [PATCH 381/405] DDP-7443 | fix not empty bug, creating fix for dynamic fields searching value --- .../java/org/broadinstitute/dsm/model/Filter.java | 12 +++++++++++- .../model/elastic/filter/query/BaseQueryBuilder.java | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/Filter.java b/src/main/java/org/broadinstitute/dsm/model/Filter.java index 7492af76e..b112d9a2e 100644 --- a/src/main/java/org/broadinstitute/dsm/model/Filter.java +++ b/src/main/java/org/broadinstitute/dsm/model/Filter.java @@ -285,7 +285,7 @@ else if (filter.isNotEmpty()) { lessThan = lessThan.substring(lessThanIndex); query += moreThan + query + lessThan; } else if (filter.isExactMatch()) { - query += EQUALS + "'#'"; + query = buildQuery(filter, query); query = query.replaceAll("#", String.valueOf(filter.getFilter1().getValue())); } else { query += " " + LIKE + " '%#%'"; @@ -301,6 +301,16 @@ else if (filter.isNotEmpty()) { return finalQuery; } + private static String buildQuery(Filter filter, String query) { + try { + Long.parseLong(String.valueOf(filter.getFilter1().getValue())); + query += EQUALS + "#"; + } catch (NumberFormatException nfe) { + query += EQUALS + "'#'"; + } + return query; + } + /** * Check if filter value is not null and is not blank * @return boolean is true if a filter is not null and not blank diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/BaseQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/BaseQueryBuilder.java index 54f9cc4df..97f693b42 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/BaseQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/BaseQueryBuilder.java @@ -60,6 +60,7 @@ protected QueryBuilder buildQueryBuilder() { } else { qb = new MatchQueryBuilder(payload.getFieldName(), dynamicFieldValues[0]); } + qb = build(qb); } else { if (jsonExtractSplitter.getDecoratedSplitter() instanceof IsNullSplitter) { qb = buildIsNullQuery(); @@ -67,7 +68,6 @@ protected QueryBuilder buildQueryBuilder() { qb = buildIsNotNullAndEmpty(); } } - qb = build(qb); break; default: throw new IllegalArgumentException(Operator.UNKNOWN_OPERATOR); From 0931d90cd99a6a5a7254271e25ac17cd93da378b Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 4 Feb 2022 16:49:28 +0400 Subject: [PATCH 382/405] DDP-7443 | add additionalType to guess field type when it comes from additional values --- .../org/broadinstitute/dsm/model/Filter.java | 18 ++++++------- .../model/participant/ParticipantWrapper.java | 25 +++++++++++++------ 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/Filter.java b/src/main/java/org/broadinstitute/dsm/model/Filter.java index b112d9a2e..57665d5b7 100644 --- a/src/main/java/org/broadinstitute/dsm/model/Filter.java +++ b/src/main/java/org/broadinstitute/dsm/model/Filter.java @@ -8,6 +8,7 @@ import java.time.format.DateTimeParseException; import java.util.Objects; +import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Getter; import lombok.NonNull; import org.apache.commons.lang3.StringUtils; @@ -88,6 +89,7 @@ public class Filter { private NameValue filter2; private String[] selectedOptions; private ParticipantColumn participantColumn; + private String additionalType; public Filter() { } @@ -285,7 +287,7 @@ else if (filter.isNotEmpty()) { lessThan = lessThan.substring(lessThanIndex); query += moreThan + query + lessThan; } else if (filter.isExactMatch()) { - query = buildQuery(filter, query); + query += NUMBER.equals(filter.additionalType) ? EQUALS + "#" : EQUALS + "'#'"; query = query.replaceAll("#", String.valueOf(filter.getFilter1().getValue())); } else { query += " " + LIKE + " '%#%'"; @@ -301,16 +303,6 @@ else if (filter.isNotEmpty()) { return finalQuery; } - private static String buildQuery(Filter filter, String query) { - try { - Long.parseLong(String.valueOf(filter.getFilter1().getValue())); - query += EQUALS + "#"; - } catch (NumberFormatException nfe) { - query += EQUALS + "'#'"; - } - return query; - } - /** * Check if filter value is not null and is not blank * @return boolean is true if a filter is not null and not blank @@ -443,4 +435,8 @@ public void setSelectedOptions(String[] selectedOptions) { public void setParticipantColumn(ParticipantColumn participantColumn) { this.participantColumn = participantColumn; } + + public void setAdditionalType(String additionalType) { + this.additionalType = additionalType; + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java index f71a36934..c8c378285 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java @@ -1,31 +1,42 @@ package org.broadinstitute.dsm.model.participant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + import com.google.common.reflect.TypeToken; import com.google.gson.Gson; import lombok.Data; import org.apache.commons.lang3.StringUtils; -import org.broadinstitute.dsm.db.*; +import org.broadinstitute.dsm.db.AbstractionActivity; +import org.broadinstitute.dsm.db.AbstractionGroup; +import org.broadinstitute.dsm.db.DDPInstance; +import org.broadinstitute.dsm.db.KitRequestShipping; +import org.broadinstitute.dsm.db.MedicalRecord; +import org.broadinstitute.dsm.db.OncHistoryDetail; +import org.broadinstitute.dsm.db.Participant; +import org.broadinstitute.dsm.db.Tissue; import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; -import org.broadinstitute.dsm.db.dao.ddp.participant.ParticipantDataDao; import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData; import org.broadinstitute.dsm.model.elastic.ESProfile; -import org.broadinstitute.dsm.model.elastic.filter.query.DsmAbstractQueryBuilder; import org.broadinstitute.dsm.model.elastic.filter.FilterParser; +import org.broadinstitute.dsm.model.elastic.filter.query.DsmAbstractQueryBuilder; import org.broadinstitute.dsm.model.elastic.search.ElasticSearch; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchParticipantDto; import org.broadinstitute.dsm.model.elastic.search.ElasticSearchable; import org.broadinstitute.dsm.model.participant.data.FamilyMemberConstants; import org.broadinstitute.dsm.statics.DBConstants; -import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.elasticsearch.index.query.BoolQueryBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; -import java.util.stream.Collectors; - @Data public class ParticipantWrapper { From 904918134189a2b66ee0f49c06eac5dc28548dab Mon Sep 17 00:00:00 2001 From: Nika Date: Mon, 7 Feb 2022 10:31:30 +0400 Subject: [PATCH 383/405] DDP-7443 | replace hardcoded data with constants, move out defining the "realm" out of the for loop --- .../org/broadinstitute/dsm/db/KitRequestShipping.java | 10 +++++----- .../broadinstitute/dsm/route/ParticipantExitRoute.java | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java index afcd927cb..5c074efa4 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java @@ -778,7 +778,7 @@ public static void deactivateKitRequest(long dsmKitRequestId, @NonNull String de kitRequestShipping.setDeactivationReason(deactivationReason); kitRequestShipping.setDeactivatedDate(deactivatedDate); - UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, "dsmKitRequestId", "dsmKitRequestId", dsmKitRequestId) + UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, ESObjectConstants.DSM_KIT_REQUEST_ID, ESObjectConstants.DSM_KIT_REQUEST_ID, dsmKitRequestId) .export(); } @@ -909,7 +909,7 @@ public static String writeRequest(@NonNull String instanceId, @NonNull String dd DDPInstanceDto ddpInstanceDto = new DDPInstanceDao().getDDPInstanceByInstanceId(Integer.valueOf(ddpInstance.getDdpInstanceId())).orElseThrow(); - UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, "dsmKitRequestId", "_id", Exportable.getParticipantGuid(ddpParticipantId, ddpInstance.getParticipantIndexES())) + UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, ESObjectConstants.DSM_KIT_REQUEST_ID, ESObjectConstants.DOC_ID, Exportable.getParticipantGuid(ddpParticipantId, ddpInstance.getParticipantIndexES())) .export(); } @@ -998,7 +998,7 @@ public static void updateKit(long dsmKitId, String userId, DDPInstanceDto ddpIns KitRequestShipping kitRequestShipping = new KitRequestShipping(null, dsmKitId, null, null, null, null); kitRequestShipping.setLabelDate(labelDate); - UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, "dsmKitId", "dsmKitId", dsmKitId) + UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, ESObjectConstants.DSM_KIT_ID, ESObjectConstants.DSM_KIT_ID, dsmKitId) .export(); } } @@ -1093,7 +1093,7 @@ public static void updateKit(String dsmKitId, Shipment participantShipment, Ship else { logger.info("Updated kit w/ dsm_kit_id " + dsmKitId, results.resultException); - UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, "dsmKitId", "dsmKitId", dsmKitId) + UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, ESObjectConstants.DSM_KIT_ID, ESObjectConstants.DSM_KIT_ID, dsmKitId) .export(); } @@ -1391,7 +1391,7 @@ public static void updateKitError(@NonNull long dsmKitRequestId, @NonNull String KitRequestShipping kitRequestShipping = new KitRequestShipping(dsmKitRequestId, null, null, null, null, message); - UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, "dsmKitRequestId", "dsmKitRequestId", dsmKitRequestId) + UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, ESObjectConstants.DSM_KIT_REQUEST_ID, ESObjectConstants.DSM_KIT_REQUEST_ID, dsmKitRequestId) .export(); } diff --git a/src/main/java/org/broadinstitute/dsm/route/ParticipantExitRoute.java b/src/main/java/org/broadinstitute/dsm/route/ParticipantExitRoute.java index 284d6e8cf..c9e44efda 100644 --- a/src/main/java/org/broadinstitute/dsm/route/ParticipantExitRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/ParticipantExitRoute.java @@ -62,6 +62,7 @@ public Object processRequest(Request request, Response response, String userId) List kitRequests = KitRequestShipping.getKitRequestsByParticipant(realm, ddpParticipantId, true); List kitsNeedAction = new ArrayList<>(); logger.info("Found " + kitRequests.size() + " kit requests"); + Optional maybeInstance = new DDPInstanceDao().getDDPInstanceByInstanceName(realm); for (KitRequestShipping kit : kitRequests) { if (kit.getScanDate() != 0 && kit.getReceiveDate() == 0) { String discardId = KitDiscard.addKitToDiscard(kit.getDsmKitRequestId(), KitDiscard.HOLD); @@ -69,7 +70,6 @@ public Object processRequest(Request request, Response response, String userId) } else { //refund label of kits which are not sent yet - Optional maybeInstance = new DDPInstanceDao().getDDPInstanceByInstanceName(realm); KitRequestShipping.refundKit(kit.getDsmKitRequestId(), DSMServer.getDDPEasypostApiKey(realm), maybeInstance.orElse(null)); } } From 64bb0bacd76f3b9f2f4b4b654e426701acf92cb4 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Mon, 7 Feb 2022 11:18:04 +0400 Subject: [PATCH 384/405] DDP-7443 | change kitType which is actually kitTypeName --- .../dsm/db/KitRequestShipping.java | 10 +++++----- .../broadinstitute/dsm/route/DashboardRoute.java | 16 ++++++++-------- .../dsm/route/KitExpressRoute.java | 4 ++-- .../dsm/route/ParticipantExitRoute.java | 2 +- .../org/broadinstitute/dsm/util/PDFAudit.java | 4 ++-- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java index 5c074efa4..002bcd0c1 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java @@ -180,7 +180,7 @@ public class KitRequestShipping extends KitRequest { private String realm; @ColumnName (DBConstants.KIT_TYPE_NAME) - private String kitType; + private String kitTypeName; @ColumnName (DBConstants.DEACTIVATION_REASON) private String deactivationReason; @@ -246,10 +246,10 @@ public class KitRequestShipping extends KitRequest { public KitRequestShipping() {} - public KitRequestShipping(String collaboratorParticipantId, String kitType, Long dsmKitRequestId, Long scanDate, Boolean error, + public KitRequestShipping(String collaboratorParticipantId, String kitTypeName, Long dsmKitRequestId, Long scanDate, Boolean error, Long receiveDate, Long deactivatedDate, String result, String upsTrackingStatus, String upsReturnStatus, String externalOrderStatus, String externalOrderNumber, Long externalOrderDate, Boolean careEvolve, String uploadReason) { - this(null, collaboratorParticipantId, null, null, null, kitType, dsmKitRequestId, null, null, null, + this(null, collaboratorParticipantId, null, null, null, kitTypeName, dsmKitRequestId, null, null, null, null, null, null, null, scanDate, error, null, receiveDate, null, deactivatedDate, null, null, null, null, null, null, externalOrderNumber, null, externalOrderStatus, null, result, @@ -275,7 +275,7 @@ public KitRequestShipping(Long dsmKitRequestId, Long dsmKitId, String easypostTo // shippingId = ddp_label !!! public KitRequestShipping(String participantId, String collaboratorParticipantId, String bspCollaboratorSampleId, String shippingId, String realm, - String kitType, Long dsmKitRequestId, Long dsmKitId, String labelUrlTo, String labelUrlReturn, + String kitTypeName, Long dsmKitRequestId, Long dsmKitId, String labelUrlTo, String labelUrlReturn, String trackingNumberTo, String trackingReturnId, String easypostTrackingToUrl, String trackingUrlReturn, Long scanDate, Boolean error, String message, Long receiveDate, String easypostAddressId, Long deactivatedDate, String deactivationReason, @@ -287,7 +287,7 @@ public KitRequestShipping(String participantId, String collaboratorParticipantId this.collaboratorParticipantId = collaboratorParticipantId; this.bspCollaboratorSampleId = bspCollaboratorSampleId; this.realm = realm; - this.kitType = kitType; + this.kitTypeName = kitTypeName; this.dsmKitId = dsmKitId; this.labelUrlTo = labelUrlTo; this.labelUrlReturn = labelUrlReturn; diff --git a/src/main/java/org/broadinstitute/dsm/route/DashboardRoute.java b/src/main/java/org/broadinstitute/dsm/route/DashboardRoute.java index 3e09a3e7e..3f7a3ab42 100644 --- a/src/main/java/org/broadinstitute/dsm/route/DashboardRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/DashboardRoute.java @@ -499,22 +499,22 @@ private void countKits(@NonNull List kits, @NonNull Set kitTypes = org.broadinstitute.dsm.model.KitType.getKitLookup(); - String key = kitRequest.getKitType() + "_" + ddpInstanceDto.getDdpInstanceId(); + String key = kitRequest.getKitTypeName() + "_" + ddpInstanceDto.getDdpInstanceId(); KitType kitType = kitTypes.get(key); Map carrierServiceTypes = @@ -143,7 +143,7 @@ private EasypostLabelRate getRateForOvernightExpress(@NonNull String kitRequestI DDPInstance ddpInstance = DDPInstance.getDDPInstance(kitRequest.getRealm()); HashMap kitTypes = org.broadinstitute.dsm.model.KitType.getKitLookup(); - String key = kitRequest.getKitType() + "_" + ddpInstance.getDdpInstanceId(); + String key = kitRequest.getKitTypeName() + "_" + ddpInstance.getDdpInstanceId(); KitType kitType = kitTypes.get(key); Map carrierServiceTypes = KitRequestSettings.getKitRequestSettings(ddpInstance.getDdpInstanceId()); diff --git a/src/main/java/org/broadinstitute/dsm/route/ParticipantExitRoute.java b/src/main/java/org/broadinstitute/dsm/route/ParticipantExitRoute.java index c9e44efda..1ea49c2db 100644 --- a/src/main/java/org/broadinstitute/dsm/route/ParticipantExitRoute.java +++ b/src/main/java/org/broadinstitute/dsm/route/ParticipantExitRoute.java @@ -66,7 +66,7 @@ public Object processRequest(Request request, Response response, String userId) for (KitRequestShipping kit : kitRequests) { if (kit.getScanDate() != 0 && kit.getReceiveDate() == 0) { String discardId = KitDiscard.addKitToDiscard(kit.getDsmKitRequestId(), KitDiscard.HOLD); - kitsNeedAction.add(new KitDiscard(discardId, kit.getKitType(), KitDiscard.HOLD)); + kitsNeedAction.add(new KitDiscard(discardId, kit.getKitTypeName(), KitDiscard.HOLD)); } else { //refund label of kits which are not sent yet diff --git a/src/main/java/org/broadinstitute/dsm/util/PDFAudit.java b/src/main/java/org/broadinstitute/dsm/util/PDFAudit.java index a7826dff6..70ad828e1 100644 --- a/src/main/java/org/broadinstitute/dsm/util/PDFAudit.java +++ b/src/main/java/org/broadinstitute/dsm/util/PDFAudit.java @@ -26,14 +26,14 @@ public void checkAndSavePDF() { // ddp with 'standard' consent and release pdfs DDPInstance instance = DDPInstance.getDDPInstanceWithRole(request.getRealm(), DBConstants.PDF_DOWNLOAD_CONSENT); //the role for release will get checked in makePDF if (instance != null && StringUtils.isNotBlank(instance.getBaseUrl()) && instance.isHasRole()) { - DDPRequestUtil.makeStandardPDF(instance, request.getParticipantId(), request.getCreatedBy(), request.getKitType()); + DDPRequestUtil.makeStandardPDF(instance, request.getParticipantId(), request.getCreatedBy(), request.getKitTypeName()); } else { // ddp without 'standard' consent and release pdfs instance = DDPInstance.getDDPInstanceWithRole(request.getRealm(), DBConstants.PDF_DOWNLOAD); //the role for release will get checked in makePDF if (instance != null && StringUtils.isNotBlank(instance.getBaseUrl()) && StringUtils.isNotBlank(instance.getParticipantIndexES()) && instance.isHasRole()) { - DDPRequestUtil.makeNonStandardPDF(instance, request.getParticipantId(), request.getCreatedBy(), request.getKitType()); + DDPRequestUtil.makeNonStandardPDF(instance, request.getParticipantId(), request.getCreatedBy(), request.getKitTypeName()); } } newAuditKit = Math.max(newAuditKit, request.getDsmKitRequestId()); From 34a43c83ba2bace4c7693643fae24a6fa43012d9 Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 8 Feb 2022 11:17:01 +0400 Subject: [PATCH 385/405] DDP-7443 | change "result" to "testResult", refactor code for migrating KitRequestShipping --- .../dsm/db/KitRequestShipping.java | 22 ++++++++++++++----- .../dsm/model/elastic/Util.java | 4 ++++ .../elastic/search/SourceMapDeserializer.java | 13 ++++++++--- .../dsm/model/elastic/UtilTest.java | 15 +++++++++++++ 4 files changed, 46 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java index 002bcd0c1..51a7e2388 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java @@ -4,6 +4,8 @@ import com.easypost.model.*; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.type.TypeReference; import lombok.Data; import lombok.NonNull; import org.apache.commons.lang3.StringUtils; @@ -30,10 +32,12 @@ import org.broadinstitute.dsm.util.EasyPostUtil; import org.broadinstitute.dsm.util.ElasticSearchUtil; import org.broadinstitute.dsm.util.KitUtil; +import org.broadinstitute.dsm.util.proxy.jackson.ObjectMapperSingleton; import org.eclipse.jetty.util.StringUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.sql.*; import java.time.Instant; import java.util.*; @@ -189,7 +193,15 @@ public class KitRequestShipping extends KitRequest { private String kitLabel; @ColumnName (DBConstants.KIT_TEST_RESULT) - private String result; + private String testResult; + + public List> getTestResult() { + try { + return ObjectMapperSingleton.instance().readValue(testResult, new TypeReference>>() {}); + } catch (IOException | NullPointerException e) { + return Collections.emptyList(); + } + } @ColumnName (DBConstants.DSM_SCAN_DATE) @DbDateConversion(SqlDateConverter.EPOCH) @@ -247,12 +259,12 @@ public class KitRequestShipping extends KitRequest { public KitRequestShipping() {} public KitRequestShipping(String collaboratorParticipantId, String kitTypeName, Long dsmKitRequestId, Long scanDate, Boolean error, - Long receiveDate, Long deactivatedDate, String result, + Long receiveDate, Long deactivatedDate, String testResult, String upsTrackingStatus, String upsReturnStatus, String externalOrderStatus, String externalOrderNumber, Long externalOrderDate, Boolean careEvolve, String uploadReason) { this(null, collaboratorParticipantId, null, null, null, kitTypeName, dsmKitRequestId, null, null, null, null, null, null, null, scanDate, error, null, receiveDate, null, deactivatedDate, null, null, null, null, null, null, externalOrderNumber, null, externalOrderStatus, null, - result, + testResult, upsTrackingStatus, upsReturnStatus, externalOrderDate, careEvolve, uploadReason, null, null, null); } @@ -280,7 +292,7 @@ public KitRequestShipping(String participantId, String collaboratorParticipantId String easypostTrackingToUrl, String trackingUrlReturn, Long scanDate, Boolean error, String message, Long receiveDate, String easypostAddressId, Long deactivatedDate, String deactivationReason, String kitLabel, Boolean express, String easypostToId, Long labelDate, String easypostShipmentStatus, - String externalOrderNumber, Boolean noReturn, String externalOrderStatus, String createdBy, String result, + String externalOrderNumber, Boolean noReturn, String externalOrderStatus, String createdBy, String testResult, String upsTrackingStatus, String upsReturnStatus, Long externalOrderDate, Boolean careEvolve, String uploadReason, String receiveDateString, String hruid, String gender) { super(dsmKitRequestId, participantId, null, shippingId, externalOrderNumber, null, externalOrderStatus, null, externalOrderDate); @@ -309,7 +321,7 @@ public KitRequestShipping(String participantId, String collaboratorParticipantId this.easypostShipmentStatus = easypostShipmentStatus; this.noReturn = noReturn; this.createdBy = createdBy; - this.result = result; + this.testResult = testResult; this.upsTrackingStatus = upsTrackingStatus; this.upsReturnStatus = upsReturnStatus; this.careEvolve = careEvolve; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 5898a33af..5221ad4e7 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -135,6 +135,10 @@ static Map convertToMap(String fieldName, Object fieldValue, Str case "follow_ups": finalResult = new HashMap<>(Map.of(underscoresToCamelCase(fieldName), new Gson().toJson(fieldValue))); break; + case "test_result": + List> testResult = ObjectMapperSingleton.readValue(String.valueOf(fieldValue), new TypeReference>>() {}); + finalResult = !testResult.isEmpty() ? Map.of(underscoresToCamelCase(fieldName), testResult.get(0)) : Map.of(); + break; case "additional_tissue_value_json": case "additional_values_json": case "data": diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java index e12f0427f..aeaf99b32 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java @@ -6,6 +6,7 @@ import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.elastic.ESDsm; import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.proxy.jackson.ObjectMapperSingleton; @@ -33,7 +34,7 @@ private Map updatePropertySourceMapIfSpecialCases(Map entry : dsmLevel.entrySet()) { outerProperty = entry.getKey(); Object outerPropertyValue = entry.getValue(); - if (!hasDynamicFields(outerProperty)) continue; + if (!hasSpecialCases(outerProperty)) continue; if (outerPropertyValue instanceof List) { List> outerPropertyValues = (List>) outerPropertyValue; List> updatedOuterPropertyValues = handleSpecialCases(outerPropertyValues);; @@ -101,12 +102,12 @@ protected Map convertDynamicFieldsFromCamelCaseToPascalCase(Map< return dynamicFields; } - private boolean hasDynamicFields(String outerProperty) { + private boolean hasSpecialCases(String outerProperty) { try { Field property = ESDsm.class.getDeclaredField(outerProperty); Class propertyType = Util.getParameterizedType(property.getGenericType()); Field[] declaredFields = propertyType.getDeclaredFields(); - return Arrays.stream(declaredFields).anyMatch(this::isDynamicField); + return Arrays.stream(declaredFields).anyMatch(field -> isDynamicField(field) || isTestResult(field)); } catch (NoSuchFieldException | ClassNotFoundException e) { throw new RuntimeException(e); } @@ -117,4 +118,10 @@ private boolean isDynamicField(Field field) { if (Objects.isNull(jsonProperty)) return false; else return jsonProperty.value().equals(ESObjectConstants.DYNAMIC_FIELDS); } + + private boolean isTestResult(Field field) { + String fieldName = field.getName(); + return fieldName.equals(DBConstants.KIT_TEST_RESULT); + } + } \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java index f00d90016..06b628f0f 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/UtilTest.java @@ -5,6 +5,7 @@ import java.lang.reflect.Field; import java.util.*; +import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.db.Participant; import org.broadinstitute.dsm.db.dao.settings.FieldSettingsDao; import org.broadinstitute.dsm.db.dto.ddp.participant.ParticipantData; @@ -95,6 +96,20 @@ public void transformJsonToMap() { assertEquals(5L, ((Map) result.get("dynamicFields")).get("longVal")); } + @Test + public void transformArrayJsonToMap() { + + setUp(); + + String json = "[{\"isCorrected\": true, \"result\": \"Negative\", \"timeCompleted\": \"2020-09-03T12:08:21.657Z\"}]"; + + Map result = Util.convertToMap("test_result", json, StringUtils.EMPTY); + + assertTrue((boolean) ((Map) result.get("testResult")).get("isCorrected")); + assertEquals("Negative", ((Map) result.get("testResult")).get("result")); + assertEquals("2020-09-03T12:08:21.657Z", ((Map) result.get("testResult")).get("timeCompleted")); + } + @Test public void getParameterizedType() throws NoSuchFieldException { class MockClass { From ae34b17d4ea01ab6ab245e0262527310b8f44f24 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 8 Feb 2022 11:48:30 +0400 Subject: [PATCH 386/405] DDP-7443 | add testResult case in SourceMapDeserializer --- .../elastic/search/SourceMapDeserializer.java | 27 ++++++++----------- .../dsm/statics/ESObjectConstants.java | 1 + .../proxy/jackson/ObjectMapperSingleton.java | 3 +++ 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java index aeaf99b32..90a9a808c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java @@ -62,6 +62,9 @@ private List> handleSpecialCases(List> o if (object.containsKey(ESObjectConstants.FOLLOW_UPS)) { clonedMap.put(ESObjectConstants.FOLLOW_UPS, convertFollowUpsJsonToList(clonedMap)); } + if (object.containsKey(ESObjectConstants.KIT_TEST_RESULT)) { + clonedMap.put(ESObjectConstants.KIT_TEST_RESULT, convertTestResultValueAsJson(clonedMap)); + } updatedOuterPropertyValues.add(clonedMap); } return updatedOuterPropertyValues; @@ -69,14 +72,7 @@ private List> handleSpecialCases(List> o private List> convertFollowUpsJsonToList(Map clonedMap) { String followUps = (String) clonedMap.get(ESObjectConstants.FOLLOW_UPS); - try { - return Objects.isNull(followUps) - ? Collections.emptyList() - : ObjectMapperSingleton.instance().readValue(followUps, new TypeReference>>() { - }); - } catch (IOException ioe) { - throw new RuntimeException(ioe); - } + return ObjectMapperSingleton.readValue(followUps, new TypeReference>>() {}); } String getDynamicFieldsValueAsJson(Map clonedMap) { @@ -84,13 +80,12 @@ String getDynamicFieldsValueAsJson(Map clonedMap) { if (ESObjectConstants.PARTICIPANT_DATA.equals(outerProperty)) { dynamicFields = convertDynamicFieldsFromCamelCaseToPascalCase(dynamicFields); } - try { - return dynamicFields.isEmpty() - ? StringUtils.EMPTY - : ObjectMapperSingleton.instance().writeValueAsString(dynamicFields); - } catch (JsonProcessingException jpe) { - throw new RuntimeException(jpe); - } + return ObjectMapperSingleton.writeValueAsString(dynamicFields); + } + + private String convertTestResultValueAsJson(Map clonedMap) { + Map testResult = (Map) clonedMap.get(ESObjectConstants.KIT_TEST_RESULT); + return ObjectMapperSingleton.writeValueAsString(List.of(testResult)); } protected Map convertDynamicFieldsFromCamelCaseToPascalCase(Map dynamicFields) { @@ -121,7 +116,7 @@ private boolean isDynamicField(Field field) { private boolean isTestResult(Field field) { String fieldName = field.getName(); - return fieldName.equals(DBConstants.KIT_TEST_RESULT); + return ESObjectConstants.KIT_TEST_RESULT.equals(fieldName); } } \ No newline at end of file diff --git a/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java b/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java index e1d51725c..713415695 100644 --- a/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java +++ b/src/main/java/org/broadinstitute/dsm/statics/ESObjectConstants.java @@ -41,6 +41,7 @@ public class ESObjectConstants { public static final String DSM_KIT_ID = "dsmKitId"; public static final String DSM_KIT_REQUEST_ID = "dsmKitRequestId"; public static final String KIT_LABEL = "kitLabel"; + public static final String KIT_TEST_RESULT = "testResult"; //common public static final String DDP_PARTICIPANT_ID = "ddpParticipantId"; diff --git a/src/main/java/org/broadinstitute/dsm/util/proxy/jackson/ObjectMapperSingleton.java b/src/main/java/org/broadinstitute/dsm/util/proxy/jackson/ObjectMapperSingleton.java index b9789a551..f3182776e 100644 --- a/src/main/java/org/broadinstitute/dsm/util/proxy/jackson/ObjectMapperSingleton.java +++ b/src/main/java/org/broadinstitute/dsm/util/proxy/jackson/ObjectMapperSingleton.java @@ -1,6 +1,8 @@ package org.broadinstitute.dsm.util.proxy.jackson; import java.io.IOException; +import java.util.Map; +import java.util.Objects; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; @@ -26,6 +28,7 @@ public static T readValue(String content, TypeReference typeReference) { } public static String writeValueAsString(Object value) { + value = Objects.isNull(value) ? Map.of() : value; try { return Helper.objectMapperInstance.writeValueAsString(value); } catch (com.fasterxml.jackson.core.JsonProcessingException e) { From 98a1c53469e5d9ac6c9287869fd313b10119e0db Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 8 Feb 2022 16:45:36 +0400 Subject: [PATCH 387/405] DDP-7443 | fix bug in SourceMapDeserializer, make tests pass, start modifying code for JSON_CONTAINS case --- .../org/broadinstitute/dsm/model/Filter.java | 6 ++-- .../dsm/model/elastic/filter/Operator.java | 3 ++ .../elastic/search/SourceMapDeserializer.java | 2 +- .../splitter/AndOrFilterSeparatorTest.java | 2 ++ .../search/SourceMapDeserializerTest.java | 36 ++++++------------- 5 files changed, 20 insertions(+), 29 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/Filter.java b/src/main/java/org/broadinstitute/dsm/model/Filter.java index 57665d5b7..d8948afa5 100644 --- a/src/main/java/org/broadinstitute/dsm/model/Filter.java +++ b/src/main/java/org/broadinstitute/dsm/model/Filter.java @@ -218,20 +218,20 @@ else if (filter.isNotEmpty()) { if (StringUtils.isNotBlank(filter.getFilter1().getValue()+"")) { //JSON_CONTAINS ( test_result , JSON_OBJECT ( 'result' , 'INVALID' ) - String notNullQuery = AND + filter.getParentName() + DBConstants.ALIAS_DELIMITER + dbElement.getColumnName() + IS_NOT_NULL; +// String notNullQuery = AND + filter.getParentName() + DBConstants.ALIAS_DELIMITER + dbElement.getColumnName() + IS_NOT_NULL; if (filter.getFilter1() != null && filter.getFilter1().getValue() != null && StringUtils.isNotBlank(String.valueOf(filter.getFilter1().getValue()))) { String quotation = ""; if (StringUtils.isNotBlank(filter.getFilter2().getValue()+"") && "'".equals(filter.getFilter2().getValue())) { quotation = "'"; } if (filter.isExactMatch()) { - query = AND + " JSON_CONTAINS ( " + filter.getParentName() + DBConstants.ALIAS_DELIMITER + dbElement.getColumnName() + " , JSON_OBJECT ( '" + filter.getFilter2().getName() + "' , "+quotation + filter.getFilter1().getValue() + quotation+" ) ) "; + query = AND + "JSON_CONTAINS ( " + filter.getParentName() + DBConstants.ALIAS_DELIMITER + dbElement.getColumnName() + " , JSON_OBJECT ( '" + filter.getFilter2().getName() + "' , "+quotation + filter.getFilter1().getValue() + quotation+" ) ) "; } else { query = AND + filter.getParentName() + DBConstants.ALIAS_DELIMITER + dbElement.getColumnName() + " -> '$[*]." + filter.getFilter2().getName() + "' like '%" + filter.getFilter1().getValue() + "%' "; } } - finalQuery = query + notNullQuery; + finalQuery = query; } } else if (CHECKBOX.equals(filter.getType())) { //1/0 diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java index 9abae8dba..60755d50b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/Operator.java @@ -20,6 +20,7 @@ public enum Operator { DATE_GREATER_THAN_EQUALS(Filter.DATE_GREATER), DATE_LESS_THAN_EQUALS(Filter.DATE_LESS), JSON_EXTRACT(Filter.JSON_EXTRACT), + JSON_CONTAINS(Filter.JSON_CONTAINS), DATE(Filter.DATE); public static final String MULTIPLE_OPTIONS_INDICATOR = "()"; @@ -63,6 +64,8 @@ public static Operator extract(String filter) { return Operator.DATE; case "NOT <=>": return Operator.DIAMOND_EQUALS; + case "JSON_CONTAINS": + return Operator.JSON_CONTAINS; case "JSON_EXTRACT =": case "JSON_EXTRACT >=": case "JSON_EXTRACT <=": diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java index 90a9a808c..b6bc6c492 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java @@ -83,7 +83,7 @@ String getDynamicFieldsValueAsJson(Map clonedMap) { return ObjectMapperSingleton.writeValueAsString(dynamicFields); } - private String convertTestResultValueAsJson(Map clonedMap) { + String convertTestResultValueAsJson(Map clonedMap) { Map testResult = (Map) clonedMap.get(ESObjectConstants.KIT_TEST_RESULT); return ObjectMapperSingleton.writeValueAsString(List.of(testResult)); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/AndOrFilterSeparatorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/AndOrFilterSeparatorTest.java index 3966697fe..030be92d5 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/AndOrFilterSeparatorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/AndOrFilterSeparatorTest.java @@ -21,6 +21,7 @@ public void parseFiltersByLogicalOperators() { "AND ( t.tissue = 'review' OR t.tissue = 'no' OR t.tissue = 'bla' ) " + "OR m.medicalRecordName = '213' " + "AND STR_TO_DATE(m.fax_sent,'%Y-%m-%d') = STR_TO_DATE('2021-12-17','%Y-%m-%d') " + + "AND JSON_CONTAINS ( k.test_result , JSON_OBJECT ( 'result' , 'result' ) ) " + "OR m.mrNotes = 'MEDICAL_RECORD_NOTESS' " + "AND m.medicalMedical = 'something AND something' " + "AND ( oD.request = 'review' OR oD.request = 'no' OR oD.request = 'bla' ) " + @@ -35,6 +36,7 @@ public void parseFiltersByLogicalOperators() { "t.tissueRecord IS NOT NULL" ,"m.dynamicFields.ragac = '55'", "( t.tissue = 'review' OR t.tissue = 'no' OR t.tissue = 'bla' )", "STR_TO_DATE(m.fax_sent,'%Y-%m-%d') = STR_TO_DATE('2021-12-17','%Y-%m-%d')", + "JSON_CONTAINS ( k.test_result , JSON_OBJECT ( 'result' , 'result' ) )", "m.medicalMedical = 'something AND something'", "( oD.request = 'review' OR oD.request = 'no' OR " + "oD.request = 'bla' )", "JSON_EXTRACT ( m.additiona`l_values_json , '$.seeingIfBugExists' )")).toArray(), diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializerTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializerTest.java index 0e7bbf967..3a96ac647 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializerTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializerTest.java @@ -1,6 +1,7 @@ package org.broadinstitute.dsm.model.elastic.search; import java.io.IOException; +import java.io.Serializable; import java.util.*; import org.broadinstitute.dsm.statics.ESObjectConstants; @@ -22,7 +23,6 @@ public void convertFollowUpsJsonToList() { ESObjectConstants.DYNAMIC_FIELDS, dynamicFields ); - SourceMapDeserializer sourceMapDeserializer = new SourceMapDeserializer(); sourceMapDeserializer.outerProperty = ESObjectConstants.PARTICIPANT_DATA; try { @@ -36,35 +36,21 @@ public void convertFollowUpsJsonToList() { } @Test - public void convertSeveralDynamicFields() { - - Map dynamicFields1 = new HashMap<>(Map.of( - "registrationType", "Self", - "registrationStatus", "Registered" - )); - - Map dynamicFields2 = new HashMap<>(Map.of( - "registrationType", "Self", - "registrationStatus", "Registered" - )); + public void testResultToJson() { - Map outerProperties1 = new HashMap<>(Map.of( - "ddpInstanceId", 12, - ESObjectConstants.DYNAMIC_FIELDS, dynamicFields1 - )); - - Map outerProperties2 = new HashMap<>(Map.of( - "ddpInstanceId", 13, - ESObjectConstants.DYNAMIC_FIELDS, dynamicFields2 - )); + SourceMapDeserializer sourceMapDeserializer = new SourceMapDeserializer(); - Map participantData = new HashMap<>(Map.of("participantData", new ArrayList<>(Arrays.asList(outerProperties1, outerProperties2)))); - Map dsm = new HashMap<>(Map.of("dsm", participantData)); + Map testResultInner = new LinkedHashMap<>(); + testResultInner.put("isCorrected", true); + testResultInner.put("timeCompleted", "2020-01-01"); + testResultInner.put("result", "Positive"); - Optional des = new SourceMapDeserializer().deserialize(dsm); + Map testResult = Map.of("testResult", testResultInner); - System.out.println(des); + String testResultValueAsJson = sourceMapDeserializer.convertTestResultValueAsJson(testResult); + String expected = "[{\"isCorrected\":true,\"timeCompleted\":\"2020-01-01\",\"result\":\"Positive\"}]"; + Assert.assertEquals(expected, testResultValueAsJson); } } \ No newline at end of file From bc03281215ad00a4f5842509e7800fc868b996c0 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 8 Feb 2022 17:15:14 +0400 Subject: [PATCH 388/405] DDP-7443 | change json array part --- src/main/java/org/broadinstitute/dsm/model/Filter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/Filter.java b/src/main/java/org/broadinstitute/dsm/model/Filter.java index d8948afa5..536acf9c0 100644 --- a/src/main/java/org/broadinstitute/dsm/model/Filter.java +++ b/src/main/java/org/broadinstitute/dsm/model/Filter.java @@ -215,7 +215,7 @@ else if (JSON_ARRAY.equals(filter.getType())) { else if (filter.isNotEmpty()) { finalQuery = query + IS_NOT_NULL + " "; } - if (StringUtils.isNotBlank(filter.getFilter1().getValue()+"")) { + else { //JSON_CONTAINS ( test_result , JSON_OBJECT ( 'result' , 'INVALID' ) // String notNullQuery = AND + filter.getParentName() + DBConstants.ALIAS_DELIMITER + dbElement.getColumnName() + IS_NOT_NULL; From 82c9388f52c08d0de35173a3b4ca20c1f5537221 Mon Sep 17 00:00:00 2001 From: Nika Date: Tue, 8 Feb 2022 17:45:19 +0400 Subject: [PATCH 389/405] DDP-7447 | add inner field name to query --- src/main/java/org/broadinstitute/dsm/model/Filter.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/Filter.java b/src/main/java/org/broadinstitute/dsm/model/Filter.java index 536acf9c0..f246f0527 100644 --- a/src/main/java/org/broadinstitute/dsm/model/Filter.java +++ b/src/main/java/org/broadinstitute/dsm/model/Filter.java @@ -208,15 +208,14 @@ else if (ADDITIONAL_VALUES.equals(filter.getType())) { finalQuery = buildJsonExtract(filter, dbElement); } else if (JSON_ARRAY.equals(filter.getType())) { - query = AND + filter.getParentName() + DBConstants.ALIAS_DELIMITER + dbElement.getColumnName() ; + // filter.getFilter2().getName() + query = AND + filter.getParentName() + DBConstants.ALIAS_DELIMITER + dbElement.getColumnName() + DBConstants.ALIAS_DELIMITER + filter.getFilter2().getName(); if (filter.isEmpty()) { finalQuery = query + IS_NULL + " "; } else if (filter.isNotEmpty()) { finalQuery = query + IS_NOT_NULL + " "; - } - else { - + } else { //JSON_CONTAINS ( test_result , JSON_OBJECT ( 'result' , 'INVALID' ) // String notNullQuery = AND + filter.getParentName() + DBConstants.ALIAS_DELIMITER + dbElement.getColumnName() + IS_NOT_NULL; if (filter.getFilter1() != null && filter.getFilter1().getValue() != null && StringUtils.isNotBlank(String.valueOf(filter.getFilter1().getValue()))) { @@ -231,7 +230,7 @@ else if (filter.isNotEmpty()) { query = AND + filter.getParentName() + DBConstants.ALIAS_DELIMITER + dbElement.getColumnName() + " -> '$[*]." + filter.getFilter2().getName() + "' like '%" + filter.getFilter1().getValue() + "%' "; } } - finalQuery = query; + finalQuery = query; } } else if (CHECKBOX.equals(filter.getType())) { //1/0 From 70f7103001c6cf07e55ade0d40d73710e960400a Mon Sep 17 00:00:00 2001 From: gmakhara Date: Tue, 8 Feb 2022 19:41:34 +0400 Subject: [PATCH 390/405] DDP-7443 | add json contains case --- .../org/broadinstitute/dsm/model/Filter.java | 5 +--- .../elastic/filter/splitter/BaseSplitter.java | 12 +++++++++ .../filter/splitter/SplitterFactory.java | 4 +++ .../splitter/IsNotNullSplitterTest.java | 27 +++++++++++++++++++ .../splitter/JsonContainsSplitterTest.java | 18 +++++++++++++ 5 files changed, 62 insertions(+), 4 deletions(-) create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/IsNotNullSplitterTest.java create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitterTest.java diff --git a/src/main/java/org/broadinstitute/dsm/model/Filter.java b/src/main/java/org/broadinstitute/dsm/model/Filter.java index f246f0527..f01ed07ff 100644 --- a/src/main/java/org/broadinstitute/dsm/model/Filter.java +++ b/src/main/java/org/broadinstitute/dsm/model/Filter.java @@ -219,10 +219,7 @@ else if (filter.isNotEmpty()) { //JSON_CONTAINS ( test_result , JSON_OBJECT ( 'result' , 'INVALID' ) // String notNullQuery = AND + filter.getParentName() + DBConstants.ALIAS_DELIMITER + dbElement.getColumnName() + IS_NOT_NULL; if (filter.getFilter1() != null && filter.getFilter1().getValue() != null && StringUtils.isNotBlank(String.valueOf(filter.getFilter1().getValue()))) { - String quotation = ""; - if (StringUtils.isNotBlank(filter.getFilter2().getValue()+"") && "'".equals(filter.getFilter2().getValue())) { - quotation = "'"; - } + String quotation = "'"; if (filter.isExactMatch()) { query = AND + "JSON_CONTAINS ( " + filter.getParentName() + DBConstants.ALIAS_DELIMITER + dbElement.getColumnName() + " , JSON_OBJECT ( '" + filter.getFilter2().getName() + "' , "+quotation + filter.getFilter1().getValue() + quotation+" ) ) "; } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/BaseSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/BaseSplitter.java index 46a1d7e91..66034e277 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/BaseSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/BaseSplitter.java @@ -1,11 +1,16 @@ package org.broadinstitute.dsm.model.elastic.filter.splitter; +import java.util.Arrays; +import java.util.stream.Collectors; + import org.apache.commons.lang3.StringUtils; import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.statics.DBConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; public abstract class BaseSplitter { + public static final int NESTED_FIELD_LEVEL = 2; protected String filter; protected String[] splittedFilter; @@ -23,6 +28,13 @@ public String getAlias() { } public String getInnerProperty() { + //k.test_result.isCompleted + if (getFieldWithAlias().length > NESTED_FIELD_LEVEL) { + return Arrays.stream(getFieldWithAlias()) + .skip(1) + .map(Util::underscoresToCamelCase) + .collect(Collectors.joining(DBConstants.ALIAS_DELIMITER)); + } return Util.underscoresToCamelCase(getFieldWithAlias()[1]); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/SplitterFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/SplitterFactory.java index 3f749e0eb..569494447 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/SplitterFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/SplitterFactory.java @@ -45,6 +45,10 @@ public static BaseSplitter createSplitter(Operator operator, String filterValue) case IS_NULL: splitter = new IsNullSplitter(); break; + case JSON_CONTAINS: +// splitter = new Json(); + splitter = null; + break; case JSON_EXTRACT: Operator decoratedOperator = Operator.extract(filterValue.replace(Filter.JSON_EXTRACT, StringUtils.EMPTY)); BaseSplitter decoratedSplitter = createSplitter(decoratedOperator, filterValue); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/IsNotNullSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/IsNotNullSplitterTest.java new file mode 100644 index 000000000..e7a0c2d73 --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/IsNotNullSplitterTest.java @@ -0,0 +1,27 @@ +package org.broadinstitute.dsm.model.elastic.filter.splitter; + +import static org.junit.Assert.*; + +import org.junit.Test; + +public class IsNotNullSplitterTest { + + + @Test + public void getNestedInnerProperty() { + BaseSplitter splitter = new IsNotNullSplitter(); + String filter = "k.test_result.isCompleted IS NOT NULL"; + splitter.setFilter(filter); + assertEquals("testResult.isCompleted", splitter.getInnerProperty()); + } + + @Test + public void getInnerProperty() { + BaseSplitter splitter = new IsNotNullSplitter(); + String filter = "k.kit_label IS NOT NULL"; + splitter.setFilter(filter); + assertEquals("kitLabel", splitter.getInnerProperty()); + } + + +} \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitterTest.java new file mode 100644 index 000000000..9ba09a3c3 --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitterTest.java @@ -0,0 +1,18 @@ +package org.broadinstitute.dsm.model.elastic.filter.splitter; + +import org.junit.Test; +import static org.junit.Assert.*; + +public class JsonContainsSplitterTest { + + @Test + public void getValue() { +// JSON_CONTAINS(k.test_result, JSON_OBJECT('isCorrected', 'true')) + BaseSplitter splitter = new JsonContainsSplitter(); + String filter = "JSON_CONTAINS(k.test_result, JSON_OBJECT('isCorrected', 'true'))"; + splitter.setFilter(filter); + assertEquals("true", splitter.getValue()[0]); + } + + +} From 73171ad5dd74cb735635d6906426841fd261d96a Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 9 Feb 2022 10:41:31 +0400 Subject: [PATCH 391/405] DDP-7443 | change test result to list --- .../java/org/broadinstitute/dsm/model/elastic/Util.java | 2 +- .../elastic/filter/splitter/JsonContainsSplitterTest.java | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java index 5221ad4e7..2237653ef 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/Util.java @@ -137,7 +137,7 @@ static Map convertToMap(String fieldName, Object fieldValue, Str break; case "test_result": List> testResult = ObjectMapperSingleton.readValue(String.valueOf(fieldValue), new TypeReference>>() {}); - finalResult = !testResult.isEmpty() ? Map.of(underscoresToCamelCase(fieldName), testResult.get(0)) : Map.of(); + finalResult = !testResult.isEmpty() ? Map.of(underscoresToCamelCase(fieldName), testResult) : Map.of(); break; case "additional_tissue_value_json": case "additional_values_json": diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitterTest.java index 9ba09a3c3..58c3a511c 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitterTest.java @@ -8,10 +8,10 @@ public class JsonContainsSplitterTest { @Test public void getValue() { // JSON_CONTAINS(k.test_result, JSON_OBJECT('isCorrected', 'true')) - BaseSplitter splitter = new JsonContainsSplitter(); - String filter = "JSON_CONTAINS(k.test_result, JSON_OBJECT('isCorrected', 'true'))"; - splitter.setFilter(filter); - assertEquals("true", splitter.getValue()[0]); +// BaseSplitter splitter = new JsonContainsSplitter(); +// String filter = "JSON_CONTAINS(k.test_result, JSON_OBJECT('isCorrected', 'true'))"; +// splitter.setFilter(filter); +// assertEquals("true", splitter.getValue()[0]); } From f5f8c9e2e7798cce750fe4fb563f09601e13f492 Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 9 Feb 2022 11:12:21 +0400 Subject: [PATCH 392/405] DDP-7443 | continue TDD for JsonContainsSplitter, make getValu() pass --- .../filter/splitter/JsonContainsSplitter.java | 39 +++++++++++++++++++ .../splitter/JsonContainsSplitterTest.java | 30 ++++++++++++-- 2 files changed, 65 insertions(+), 4 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitter.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitter.java new file mode 100644 index 000000000..9da17cb5a --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitter.java @@ -0,0 +1,39 @@ +package org.broadinstitute.dsm.model.elastic.filter.splitter; + +import org.apache.commons.lang3.StringUtils; +import org.broadinstitute.dsm.model.Filter; +import org.broadinstitute.dsm.model.elastic.Util; +import org.broadinstitute.dsm.statics.DBConstants; +import org.broadinstitute.dsm.statics.ESObjectConstants; +import org.broadinstitute.dsm.util.ElasticSearchUtil; + +public class JsonContainsSplitter extends BaseSplitter { + + @Override + public String[] split() { + // k.test_result, JSON_OBJECT'isCorrected', 'true' + // ["k.test_result," "'isCorrected', 'true'"] + // + return filter.split(Filter.JSON_CONTAINS)[1] + .replace(Filter.OPEN_PARENTHESIS, StringUtils.EMPTY) + .replace(Filter.CLOSE_PARENTHESIS,StringUtils.EMPTY) + .split(Filter.JSON_OBJECT); + + } + + @Override + public String[] getValue() { + return new String[] {splittedFilter[1].split(Util.COMMA_SEPARATOR)[1].trim()}; + } + + @Override + public String getInnerProperty() { + + return super.getInnerProperty(); + } + + @Override + protected String[] getFieldWithAlias() { + return splittedFilter[0].split(ElasticSearchUtil.ESCAPE_CHARACTER_DOT_SEPARATOR); + } +} diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitterTest.java index 58c3a511c..12afa6d05 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitterTest.java @@ -8,11 +8,33 @@ public class JsonContainsSplitterTest { @Test public void getValue() { // JSON_CONTAINS(k.test_result, JSON_OBJECT('isCorrected', 'true')) -// BaseSplitter splitter = new JsonContainsSplitter(); -// String filter = "JSON_CONTAINS(k.test_result, JSON_OBJECT('isCorrected', 'true'))"; -// splitter.setFilter(filter); -// assertEquals("true", splitter.getValue()[0]); + BaseSplitter splitter = new JsonContainsSplitter(); + String filter = "JSON_CONTAINS(k.test_result, JSON_OBJECT('isCorrected', 'true'))"; + splitter.setFilter(filter); + assertEquals("'true'", splitter.getValue()[0]); } + @Test + public void getInnerProperty() { +// JSON_CONTAINS(k.test_result, JSON_OBJECT('isCorrected', 'true')) + BaseSplitter splitter = new JsonContainsSplitter(); + String filter = "JSON_CONTAINS(k.test_result, JSON_OBJECT('isCorrected', 'true'))"; + splitter.setFilter(filter); + assertEquals("testResult.isCorrected", splitter.getInnerProperty()); + } + + @Test + public void getFieldWithAlias() { +// JSON_CONTAINS(k.test_result, JSON_OBJECT('isCorrected', 'true')) + // [k, test_result] + BaseSplitter splitter = new JsonContainsSplitter(); + String filter = "JSON_CONTAINS(k.test_result, JSON_OBJECT('isCorrected', 'true'))"; + splitter.setFilter(filter); + assertEquals("k", splitter.getFieldWithAlias()[0]); + assertEquals("test_result", splitter.getFieldWithAlias()[1]); + } + + + } From 337b0d35dea2440ef5911c96266841e637eeff56 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 9 Feb 2022 11:43:04 +0400 Subject: [PATCH 393/405] DDP-7443 | start creating TestResultCollectionBuilder --- .../elastic/filter/splitter/BaseSplitter.java | 1 - .../filter/splitter/JsonContainsSplitter.java | 14 +++++++------ .../filter/splitter/SplitterFactory.java | 3 +-- .../TestResultCollectionQueryBuilderTest.java | 21 +++++++++++++++++++ .../splitter/JsonContainsSplitterTest.java | 6 +++--- 5 files changed, 33 insertions(+), 12 deletions(-) create mode 100644 src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/TestResultCollectionQueryBuilderTest.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/BaseSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/BaseSplitter.java index 66034e277..d422ec8c8 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/BaseSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/BaseSplitter.java @@ -28,7 +28,6 @@ public String getAlias() { } public String getInnerProperty() { - //k.test_result.isCompleted if (getFieldWithAlias().length > NESTED_FIELD_LEVEL) { return Arrays.stream(getFieldWithAlias()) .skip(1) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitter.java index 9da17cb5a..3e5d3f8d9 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitter.java @@ -11,9 +11,6 @@ public class JsonContainsSplitter extends BaseSplitter { @Override public String[] split() { - // k.test_result, JSON_OBJECT'isCorrected', 'true' - // ["k.test_result," "'isCorrected', 'true'"] - // return filter.split(Filter.JSON_CONTAINS)[1] .replace(Filter.OPEN_PARENTHESIS, StringUtils.EMPTY) .replace(Filter.CLOSE_PARENTHESIS,StringUtils.EMPTY) @@ -28,12 +25,17 @@ public String[] getValue() { @Override public String getInnerProperty() { - - return super.getInnerProperty(); + String nestedProperty = splittedFilter[1] + .split(Util.COMMA_SEPARATOR)[0] + .replaceAll(Filter.SINGLE_QUOTE, StringUtils.EMPTY); + return String.join(DBConstants.ALIAS_DELIMITER, Util.underscoresToCamelCase(getFieldWithAlias()[1]), nestedProperty); } @Override protected String[] getFieldWithAlias() { - return splittedFilter[0].split(ElasticSearchUtil.ESCAPE_CHARACTER_DOT_SEPARATOR); + return splittedFilter[0] + .replace(Util.COMMA_SEPARATOR, StringUtils.EMPTY) + .trim() + .split(ElasticSearchUtil.ESCAPE_CHARACTER_DOT_SEPARATOR); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/SplitterFactory.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/SplitterFactory.java index 569494447..fc15b9d4a 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/SplitterFactory.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/SplitterFactory.java @@ -46,8 +46,7 @@ public static BaseSplitter createSplitter(Operator operator, String filterValue) splitter = new IsNullSplitter(); break; case JSON_CONTAINS: -// splitter = new Json(); - splitter = null; + splitter = new JsonContainsSplitter(); break; case JSON_EXTRACT: Operator decoratedOperator = Operator.extract(filterValue.replace(Filter.JSON_EXTRACT, StringUtils.EMPTY)); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/TestResultCollectionQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/TestResultCollectionQueryBuilderTest.java new file mode 100644 index 000000000..e811e6b1a --- /dev/null +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/TestResultCollectionQueryBuilderTest.java @@ -0,0 +1,21 @@ +package org.broadinstitute.dsm.model.elastic.filter.query; + +import org.broadinstitute.dsm.model.elastic.filter.FilterParser; +import org.broadinstitute.dsm.model.elastic.filter.Operator; +import org.broadinstitute.dsm.model.elastic.filter.splitter.JsonContainsSplitter; +import org.elasticsearch.index.query.QueryBuilder; +import org.junit.Test; + +public class TestResultCollectionQueryBuilderTest { + + + @Test + public void build() { + BaseQueryBuilder queryBuilder = new TestResultCollectionQueryBuilder(); + QueryPayload queryPayload = + new QueryPayload("dsm.kitRequestShipping", "testResult.isCorrected", new FilterParser().parse(new String[] {"'true'"})); + QueryBuilder query = queryBuilder.buildEachQuery(Operator.JSON_CONTAINS, queryPayload, new JsonContainsSplitter()); + + } + +} diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitterTest.java index 12afa6d05..7f674152d 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitterTest.java @@ -7,7 +7,7 @@ public class JsonContainsSplitterTest { @Test public void getValue() { -// JSON_CONTAINS(k.test_result, JSON_OBJECT('isCorrected', 'true')) + //JSON_CONTAINS(k.test_result, JSON_OBJECT('isCorrected', 'true')) BaseSplitter splitter = new JsonContainsSplitter(); String filter = "JSON_CONTAINS(k.test_result, JSON_OBJECT('isCorrected', 'true'))"; splitter.setFilter(filter); @@ -16,7 +16,7 @@ public void getValue() { @Test public void getInnerProperty() { -// JSON_CONTAINS(k.test_result, JSON_OBJECT('isCorrected', 'true')) + //JSON_CONTAINS(k.test_result, JSON_OBJECT('isCorrected', 'true')) BaseSplitter splitter = new JsonContainsSplitter(); String filter = "JSON_CONTAINS(k.test_result, JSON_OBJECT('isCorrected', 'true'))"; splitter.setFilter(filter); @@ -25,7 +25,7 @@ public void getInnerProperty() { @Test public void getFieldWithAlias() { -// JSON_CONTAINS(k.test_result, JSON_OBJECT('isCorrected', 'true')) + //JSON_CONTAINS(k.test_result, JSON_OBJECT('isCorrected', 'true')) // [k, test_result] BaseSplitter splitter = new JsonContainsSplitter(); String filter = "JSON_CONTAINS(k.test_result, JSON_OBJECT('isCorrected', 'true'))"; From 364be1ff9240b88e7760d26dabe14cd03d90b99f Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 9 Feb 2022 16:20:19 +0400 Subject: [PATCH 394/405] DDP-7443 | make TestResultCollectionQueryBuilder for KitRequestShipping test results --- .../elastic/filter/query/BaseQueryBuilder.java | 1 + .../model/elastic/filter/query/QueryPayload.java | 4 ++++ .../query/TestResultCollectionQueryBuilder.java | 15 +++++++++++++++ .../elastic/filter/splitter/BaseSplitter.java | 4 ++++ .../filter/splitter/JsonContainsSplitter.java | 3 +-- .../TestResultCollectionQueryBuilderTest.java | 14 ++++++++++++-- .../filter/splitter/JsonContainsSplitterTest.java | 13 ++++++++++++- 7 files changed, 49 insertions(+), 5 deletions(-) create mode 100644 src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/TestResultCollectionQueryBuilder.java diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/BaseQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/BaseQueryBuilder.java index 97f693b42..11a82f459 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/BaseQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/BaseQueryBuilder.java @@ -18,6 +18,7 @@ protected QueryBuilder buildQueryBuilder() { case EQUALS: case DATE: case DIAMOND_EQUALS: + case JSON_CONTAINS: case STR_DATE: qb = build(new MatchQueryBuilder(payload.getFieldName(), payload.getValues()[0])); break; diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryPayload.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryPayload.java index 88b6cf36e..7f6b7d374 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryPayload.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/QueryPayload.java @@ -19,4 +19,8 @@ public QueryPayload(String path, String property, Object[] values) { public String getFieldName() { return path + DBConstants.ALIAS_DELIMITER + property; } + + public void setPath(String path) { + this.path = path; + } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/TestResultCollectionQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/TestResultCollectionQueryBuilder.java new file mode 100644 index 000000000..3e91c5918 --- /dev/null +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/TestResultCollectionQueryBuilder.java @@ -0,0 +1,15 @@ +package org.broadinstitute.dsm.model.elastic.filter.query; + +import org.broadinstitute.dsm.model.elastic.filter.Operator; +import org.broadinstitute.dsm.model.elastic.filter.splitter.BaseSplitter; +import org.broadinstitute.dsm.statics.DBConstants; +import org.elasticsearch.index.query.QueryBuilder; + +public class TestResultCollectionQueryBuilder extends CollectionQueryBuilder { + + @Override + public QueryBuilder buildEachQuery(Operator operator, QueryPayload queryPayload, BaseSplitter splitter) { + queryPayload.setPath(String.join(DBConstants.ALIAS_DELIMITER, queryPayload.getPath(), splitter.getField())); + return super.buildEachQuery(operator, queryPayload, splitter); + } +} diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/BaseSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/BaseSplitter.java index d422ec8c8..acd7ab118 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/BaseSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/BaseSplitter.java @@ -37,6 +37,10 @@ public String getInnerProperty() { return Util.underscoresToCamelCase(getFieldWithAlias()[1]); } + public String getField() { + return Util.underscoresToCamelCase(getFieldWithAlias()[1]); + } + protected String[] getFieldWithAlias() { return splittedFilter[0].trim().split(ElasticSearchUtil.ESCAPE_CHARACTER_DOT_SEPARATOR); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitter.java index 3e5d3f8d9..32f503f1b 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitter.java @@ -25,10 +25,9 @@ public String[] getValue() { @Override public String getInnerProperty() { - String nestedProperty = splittedFilter[1] + return splittedFilter[1] .split(Util.COMMA_SEPARATOR)[0] .replaceAll(Filter.SINGLE_QUOTE, StringUtils.EMPTY); - return String.join(DBConstants.ALIAS_DELIMITER, Util.underscoresToCamelCase(getFieldWithAlias()[1]), nestedProperty); } @Override diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/TestResultCollectionQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/TestResultCollectionQueryBuilderTest.java index e811e6b1a..20a3413a8 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/TestResultCollectionQueryBuilderTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/TestResultCollectionQueryBuilderTest.java @@ -1,9 +1,13 @@ package org.broadinstitute.dsm.model.elastic.filter.query; +import org.apache.lucene.search.join.ScoreMode; import org.broadinstitute.dsm.model.elastic.filter.FilterParser; import org.broadinstitute.dsm.model.elastic.filter.Operator; import org.broadinstitute.dsm.model.elastic.filter.splitter.JsonContainsSplitter; +import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.index.query.NestedQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; +import org.junit.Assert; import org.junit.Test; public class TestResultCollectionQueryBuilderTest { @@ -13,9 +17,15 @@ public class TestResultCollectionQueryBuilderTest { public void build() { BaseQueryBuilder queryBuilder = new TestResultCollectionQueryBuilder(); QueryPayload queryPayload = - new QueryPayload("dsm.kitRequestShipping", "testResult.isCorrected", new FilterParser().parse(new String[] {"'true'"})); - QueryBuilder query = queryBuilder.buildEachQuery(Operator.JSON_CONTAINS, queryPayload, new JsonContainsSplitter()); + new QueryPayload("dsm.kitRequestShipping", "isCorrected", new FilterParser().parse(new String[] {"'true'"})); + String filter = "JSON_CONTAINS(k.test_result, JSON_OBJECT('isCorrected', 'true'))"; + JsonContainsSplitter splitter = new JsonContainsSplitter(); + splitter.setFilter(filter); + QueryBuilder query = queryBuilder.buildEachQuery(Operator.JSON_CONTAINS, queryPayload, splitter); + MatchQueryBuilder matchQueryBuilder = new MatchQueryBuilder("dsm.kitRequestShipping.testResult.isCorrected", true); + NestedQueryBuilder expected = new NestedQueryBuilder("dsm.kitRequestShipping.testResult", matchQueryBuilder, ScoreMode.Avg); + Assert.assertEquals(expected, query); } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitterTest.java index 7f674152d..b5a9b2e53 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitterTest.java @@ -1,7 +1,8 @@ package org.broadinstitute.dsm.model.elastic.filter.splitter; import org.junit.Test; -import static org.junit.Assert.*; + +import static org.junit.Assert.assertEquals; public class JsonContainsSplitterTest { @@ -34,6 +35,16 @@ public void getFieldWithAlias() { assertEquals("test_result", splitter.getFieldWithAlias()[1]); } + @Test + public void getField() { + BaseSplitter splitter = new JsonContainsSplitter(); + String filter = "JSON_CONTAINS(k.test_result, JSON_OBJECT('isCorrected', 'true'))"; + splitter.setFilter(filter); + assertEquals("testResult", splitter.getField()); + } + + + From d216fff41529231aaf28699bb313408d26482370 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 9 Feb 2022 16:50:42 +0400 Subject: [PATCH 395/405] DDP-7443 | add factory method to BaseQueryBuilder --- .../elastic/filter/query/BaseQueryBuilder.java | 14 ++++++++++++++ .../filter/query/DsmAbstractQueryBuilder.java | 11 ++--------- .../query/TestResultCollectionQueryBuilder.java | 4 +++- .../elastic/filter/splitter/BaseSplitter.java | 2 +- .../dsm/model/participant/ParticipantWrapper.java | 2 +- .../broadinstitute/dsm/model/elastic/UtilTest.java | 7 ++++--- .../elastic/filter/query/BaseQueryBuilderTest.java | 14 ++++++++++++-- .../filter/query/CollectionQueryBuilderTest.java | 2 +- .../filter/query/SingleQueryBuilderTest.java | 2 +- .../filter/splitter/JsonContainsSplitterTest.java | 2 +- 10 files changed, 40 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/BaseQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/BaseQueryBuilder.java index 11a82f459..82199aa1c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/BaseQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/BaseQueryBuilder.java @@ -1,6 +1,7 @@ package org.broadinstitute.dsm.model.elastic.filter.query; import org.apache.commons.lang3.StringUtils; +import org.broadinstitute.dsm.model.elastic.Util; import org.broadinstitute.dsm.model.elastic.filter.Operator; import org.broadinstitute.dsm.model.elastic.filter.splitter.*; import org.elasticsearch.index.query.*; @@ -99,4 +100,17 @@ public QueryBuilder buildEachQuery(Operator operator, return buildQueryBuilder(); } + public static BaseQueryBuilder of(String alias, String fieldName) { + BaseQueryBuilder queryBuilder; + boolean isCollection = Util.TABLE_ALIAS_MAPPINGS.get(alias).isCollection(); + if (isCollection) { + if (TestResultCollectionQueryBuilder.TEST_RESULT.equals(fieldName)) queryBuilder = + new TestResultCollectionQueryBuilder(); + else queryBuilder = new CollectionQueryBuilder(); + } else { + queryBuilder = new SingleQueryBuilder(); + } + return queryBuilder; + } + } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/DsmAbstractQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/DsmAbstractQueryBuilder.java index 46f87cc6c..2a68ef4b8 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/DsmAbstractQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/DsmAbstractQueryBuilder.java @@ -28,15 +28,7 @@ public class DsmAbstractQueryBuilder { protected AndOrFilterSeparator filterSeparator; private BaseQueryBuilder baseQueryBuilder; - public static DsmAbstractQueryBuilder of(String alias) { - boolean isCollection = Util.TABLE_ALIAS_MAPPINGS.get(alias).isCollection(); - return isCollection - ? new DsmAbstractQueryBuilder(new CollectionQueryBuilder()) - : new DsmAbstractQueryBuilder(new SingleQueryBuilder()); - } - - private DsmAbstractQueryBuilder(BaseQueryBuilder baseQueryBuilder) { - this.baseQueryBuilder = baseQueryBuilder; + public DsmAbstractQueryBuilder() { boolQueryBuilder = new BoolQueryBuilder(); } @@ -67,6 +59,7 @@ protected void buildUpQuery(List filterValues, FilterStrategy filterStra Operator operator = Operator.extract(filterValue); splitter = SplitterFactory.createSplitter(operator, filterValue); splitter.setFilter(filterValue); + baseQueryBuilder = BaseQueryBuilder.of(splitter.getAlias(), splitter.getFieldName()); QueryPayload queryPayload = new QueryPayload(buildPath(), splitter.getInnerProperty(), parser.parse(splitter.getValue())); filterStrategy.build(boolQueryBuilder, baseQueryBuilder.buildEachQuery(operator, queryPayload, splitter)); } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/TestResultCollectionQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/TestResultCollectionQueryBuilder.java index 3e91c5918..01779a844 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/TestResultCollectionQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/TestResultCollectionQueryBuilder.java @@ -7,9 +7,11 @@ public class TestResultCollectionQueryBuilder extends CollectionQueryBuilder { + public static final String TEST_RESULT = "testResult"; + @Override public QueryBuilder buildEachQuery(Operator operator, QueryPayload queryPayload, BaseSplitter splitter) { - queryPayload.setPath(String.join(DBConstants.ALIAS_DELIMITER, queryPayload.getPath(), splitter.getField())); + queryPayload.setPath(String.join(DBConstants.ALIAS_DELIMITER, queryPayload.getPath(), splitter.getFieldName())); return super.buildEachQuery(operator, queryPayload, splitter); } } diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/BaseSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/BaseSplitter.java index acd7ab118..73c772c9f 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/BaseSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/BaseSplitter.java @@ -37,7 +37,7 @@ public String getInnerProperty() { return Util.underscoresToCamelCase(getFieldWithAlias()[1]); } - public String getField() { + public String getFieldName() { return Util.underscoresToCamelCase(getFieldWithAlias()[1]); } diff --git a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java index c8c378285..84c7b4e05 100644 --- a/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java +++ b/src/main/java/org/broadinstitute/dsm/model/participant/ParticipantWrapper.java @@ -94,7 +94,7 @@ private void fetchAndPrepareDataByFilters(DDPInstance ddpInstance, Map result = Util.convertToMap("test_result", json, StringUtils.EMPTY); - assertTrue((boolean) ((Map) result.get("testResult")).get("isCorrected")); - assertEquals("Negative", ((Map) result.get("testResult")).get("result")); - assertEquals("2020-09-03T12:08:21.657Z", ((Map) result.get("testResult")).get("timeCompleted")); + Map testResult = (Map) ((List) result.get("testResult")).get(0); + assertTrue((boolean) testResult.get("isCorrected")); + assertEquals("Negative", testResult.get("result")); + assertEquals("2020-09-03T12:08:21.657Z", testResult.get("timeCompleted")); } @Test diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/BaseQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/BaseQueryBuilderTest.java index 4aef40934..78b1621be 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/BaseQueryBuilderTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/BaseQueryBuilderTest.java @@ -1,13 +1,13 @@ package org.broadinstitute.dsm.model.elastic.filter.query; +import static org.junit.Assert.assertTrue; + import org.broadinstitute.dsm.model.elastic.filter.Operator; import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.index.query.NestedQueryBuilder; import org.elasticsearch.index.query.RangeQueryBuilder; import org.junit.Test; -import static org.junit.Assert.assertTrue; - public class BaseQueryBuilderTest { @@ -37,4 +37,14 @@ public void buildQueryBuilder() { assertTrue(queryBuilder.query() instanceof RangeQueryBuilder); } + @Test + public void of() { + BaseQueryBuilder testResultCollectionQueryBuilder = BaseQueryBuilder.of("k", "testResult"); + BaseQueryBuilder collectionQueryBuilder = BaseQueryBuilder.of("m", ""); + BaseQueryBuilder singleQueryBuilder = BaseQueryBuilder.of("o", ""); + assertTrue(testResultCollectionQueryBuilder instanceof TestResultCollectionQueryBuilder); + assertTrue(collectionQueryBuilder instanceof CollectionQueryBuilder); + assertTrue(singleQueryBuilder instanceof SingleQueryBuilder); + } + } \ No newline at end of file diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/CollectionQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/CollectionQueryBuilderTest.java index 70f7c65b1..d4914c133 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/CollectionQueryBuilderTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/CollectionQueryBuilderTest.java @@ -15,7 +15,7 @@ public class CollectionQueryBuilderTest { @Before public void setUp() { - dsmAbstractQueryBuilder = DsmAbstractQueryBuilder.of("m"); + dsmAbstractQueryBuilder = new DsmAbstractQueryBuilder(); dsmAbstractQueryBuilder.setParser(new FilterParser()); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/SingleQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/SingleQueryBuilderTest.java index 6936def9e..eaf06d03d 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/SingleQueryBuilderTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/SingleQueryBuilderTest.java @@ -14,7 +14,7 @@ public class SingleQueryBuilderTest { @Before public void setUp() { - dsmAbstractQueryBuilder = DsmAbstractQueryBuilder.of("p"); + dsmAbstractQueryBuilder = new DsmAbstractQueryBuilder(); dsmAbstractQueryBuilder.setParser(new FilterParser()); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitterTest.java index b5a9b2e53..9164f46f7 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitterTest.java @@ -40,7 +40,7 @@ public void getField() { BaseSplitter splitter = new JsonContainsSplitter(); String filter = "JSON_CONTAINS(k.test_result, JSON_OBJECT('isCorrected', 'true'))"; splitter.setFilter(filter); - assertEquals("testResult", splitter.getField()); + assertEquals("testResult", splitter.getFieldName()); } From 5181336a4f1f518c5f386ddc2084839b284bb3f0 Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 9 Feb 2022 17:21:05 +0400 Subject: [PATCH 396/405] DDP-7443 | fix tests --- .../elastic/filter/query/CollectionQueryBuilderTest.java | 6 ++---- .../elastic/filter/splitter/AndOrFilterSeparatorTest.java | 2 +- .../elastic/filter/splitter/JsonContainsSplitterTest.java | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/CollectionQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/CollectionQueryBuilderTest.java index d4914c133..b072fb67c 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/CollectionQueryBuilderTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/CollectionQueryBuilderTest.java @@ -133,13 +133,11 @@ public void dynamicFieldsQueryBuild() { dsmAbstractQueryBuilder.setFilter(filter); AbstractQueryBuilder actual = dsmAbstractQueryBuilder.build(); - BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder(); - boolQueryBuilder.must(new ExistsQueryBuilder("dsm.medicalRecord.dynamicFields.tryAgain")); + boolQueryBuilder.must(new NestedQueryBuilder("dsm.medicalRecord", new ExistsQueryBuilder("dsm.medicalRecord.dynamicFields.tryAgain"), ScoreMode.Avg)); AbstractQueryBuilder expected = new BoolQueryBuilder().must(new NestedQueryBuilder("dsm.medicalRecord", - new MatchQueryBuilder("dsm.medicalRecord.dynamicFields.seeingIfBugExists", true), ScoreMode.Avg)) - .must(new NestedQueryBuilder("dsm.medicalRecord", boolQueryBuilder ,ScoreMode.Avg)); + new MatchQueryBuilder("dsm.medicalRecord.dynamicFields.seeingIfBugExists", true), ScoreMode.Avg)).must(boolQueryBuilder); Assert.assertEquals(expected, actual); } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/AndOrFilterSeparatorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/AndOrFilterSeparatorTest.java index 030be92d5..9475d834d 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/AndOrFilterSeparatorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/AndOrFilterSeparatorTest.java @@ -21,7 +21,7 @@ public void parseFiltersByLogicalOperators() { "AND ( t.tissue = 'review' OR t.tissue = 'no' OR t.tissue = 'bla' ) " + "OR m.medicalRecordName = '213' " + "AND STR_TO_DATE(m.fax_sent,'%Y-%m-%d') = STR_TO_DATE('2021-12-17','%Y-%m-%d') " + - "AND JSON_CONTAINS ( k.test_result , JSON_OBJECT ( 'result' , 'result' ) ) " + + "AND JSON_CONTAINS ( k.test_result , JSON_OBJECT ( 'result' , 'result' ) ) " + "OR m.mrNotes = 'MEDICAL_RECORD_NOTESS' " + "AND m.medicalMedical = 'something AND something' " + "AND ( oD.request = 'review' OR oD.request = 'no' OR oD.request = 'bla' ) " + diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitterTest.java index 9164f46f7..567d89982 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitterTest.java @@ -21,7 +21,7 @@ public void getInnerProperty() { BaseSplitter splitter = new JsonContainsSplitter(); String filter = "JSON_CONTAINS(k.test_result, JSON_OBJECT('isCorrected', 'true'))"; splitter.setFilter(filter); - assertEquals("testResult.isCorrected", splitter.getInnerProperty()); + assertEquals("isCorrected", splitter.getInnerProperty()); } @Test From 9208da936e90526288244ca7b404fef443a9ab65 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Wed, 9 Feb 2022 17:52:46 +0400 Subject: [PATCH 397/405] DDP-7443 | adding adding more upsert facade classes for kit updating --- .../dsm/model/KitRequestExternal.java | 23 +++++++++++++++++-- .../util/externalShipper/GBFRequestUtil.java | 6 +++-- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/KitRequestExternal.java b/src/main/java/org/broadinstitute/dsm/model/KitRequestExternal.java index 0f3814066..76ea7d3c2 100644 --- a/src/main/java/org/broadinstitute/dsm/model/KitRequestExternal.java +++ b/src/main/java/org/broadinstitute/dsm/model/KitRequestExternal.java @@ -1,8 +1,13 @@ package org.broadinstitute.dsm.model; import lombok.NonNull; -import org.broadinstitute.ddp.db.SimpleResult; +import org.broadinstitute.dsm.db.KitRequestShipping; +import org.broadinstitute.dsm.db.dao.ddp.instance.DDPInstanceDao; +import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; import org.broadinstitute.dsm.model.ddp.DDPParticipant; +import org.broadinstitute.dsm.model.elastic.export.painless.UpsertPainlessFacade; +import org.broadinstitute.dsm.statics.DBConstants; +import org.broadinstitute.dsm.statics.ESObjectConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,7 +35,8 @@ public KitRequestExternal(long dsmKitRequestId, String participantId, String sho } // update kit request with status and date of external shipper - public static void updateKitRequest(Connection conn, String externalOrderStatus, long externalOrderDate, String dsmKitRequestId) { + public static void updateKitRequest(Connection conn, String externalOrderStatus, long externalOrderDate, String dsmKitRequestId, + int instanceId) { try (PreparedStatement stmt = conn.prepareStatement(SQL_UPDATE_KIT_REQUEST_EXTERNAL_SHIPPER_STATUS)) { stmt.setString(1, externalOrderStatus); stmt.setLong(2, externalOrderDate); @@ -45,6 +51,13 @@ public static void updateKitRequest(Connection conn, String externalOrderStatus, catch (Exception e) { throw new RuntimeException("Error updating kit request w/ dsm_kit_request_id " + dsmKitRequestId, e); } + KitRequestShipping kitRequestShipping = new KitRequestShipping(); + kitRequestShipping.setDsmKitRequestId(Long.valueOf(dsmKitRequestId)); + kitRequestShipping.setExternalOrderStatus(externalOrderStatus); + kitRequestShipping.setExternalOrderDate(externalOrderDate); + DDPInstanceDto ddpInstanceDto = new DDPInstanceDao().getDDPInstanceByInstanceId(instanceId).orElseThrow(); + UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, ESObjectConstants.DSM_KIT_REQUEST_ID, + ESObjectConstants.DSM_KIT_REQUEST_ID, dsmKitRequestId); } // update kit request with response of external shipper @@ -82,5 +95,11 @@ public static void updateKitRequestResponse(@NonNull Connection conn, String tra catch (Exception e) { logger.error("Error updating kit w/ dsm_kit_request_id " + dsmKitRequestId, e); } + KitRequestShipping kitRequestShipping = new KitRequestShipping(); + kitRequestShipping.setDsmKitRequestId(Long.valueOf(dsmKitRequestId)); + kitRequestShipping.setTrac(Long.valueOf(dsmKitRequestId)); + kitRequestShipping.setDsmKitRequestId(Long.valueOf(dsmKitRequestId)); + kitRequestShipping.setDsmKitRequestId(Long.valueOf(dsmKitRequestId)); + } } diff --git a/src/main/java/org/broadinstitute/dsm/util/externalShipper/GBFRequestUtil.java b/src/main/java/org/broadinstitute/dsm/util/externalShipper/GBFRequestUtil.java index 3a3caf9a3..8010a4574 100644 --- a/src/main/java/org/broadinstitute/dsm/util/externalShipper/GBFRequestUtil.java +++ b/src/main/java/org/broadinstitute/dsm/util/externalShipper/GBFRequestUtil.java @@ -243,7 +243,8 @@ public void orderStatus(Connection conn, KitRequest kit, int instanceId, boolean List dsmKitRequestIds = getDSMKitRequestIds(status.getOrderNumber()); if (dsmKitRequestIds != null && !dsmKitRequestIds.isEmpty()) { for (String dsmKitRequestId : dsmKitRequestIds) { - KitRequestExternal.updateKitRequest(conn, status.getOrderStatus(), System.currentTimeMillis(), dsmKitRequestId);// in order to update time for the next 24 hour check we need this + KitRequestExternal.updateKitRequest(conn, status.getOrderStatus(), System.currentTimeMillis(), + dsmKitRequestId, instanceId);// in order to update time for the next 24 hour check we need this } } logger.warn("Kit Request with external order number " + kit.getExternalOrderNumber() + "has not been shipped in the last 24 hours! ");//todo pegah uncomment for production @@ -280,7 +281,8 @@ else if (status.getOrderStatus().contains("CANCELLED") && (StringUtils.isBlank(k List dsmKitRequestIds = getDSMKitRequestIds(status.getOrderNumber()); if (dsmKitRequestIds != null && !dsmKitRequestIds.isEmpty()) { for (String dsmKitRequestId : dsmKitRequestIds) { - KitRequestExternal.updateKitRequest(conn, status.getOrderStatus(), System.currentTimeMillis(), dsmKitRequestId); + KitRequestExternal.updateKitRequest(conn, status.getOrderStatus(), System.currentTimeMillis(), + dsmKitRequestId, instanceId); } } } From 49a374f1f73d178ddab4d7c478d3734b634b8931 Mon Sep 17 00:00:00 2001 From: Nika Date: Wed, 9 Feb 2022 18:17:24 +0400 Subject: [PATCH 398/405] DDP-7443 | add UpsertPainlessExport for GBFRequestUtil.java --- .../broadinstitute/dsm/db/KitRequestShipping.java | 15 +++++++-------- .../dsm/jobs/ExternalShipperJob.java | 2 +- .../dsm/model/KitRequestExternal.java | 14 ++++++++++---- .../dsm/util/externalShipper/ExternalShipper.java | 3 ++- .../dsm/util/externalShipper/GBFRequestUtil.java | 8 ++++---- src/test/java/org/broadinstitute/dsm/GBFTest.java | 2 +- 6 files changed, 25 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java index 51a7e2388..6da70349f 100644 --- a/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java +++ b/src/main/java/org/broadinstitute/dsm/db/KitRequestShipping.java @@ -4,7 +4,6 @@ import com.easypost.model.*; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.type.TypeReference; import lombok.Data; import lombok.NonNull; @@ -162,7 +161,7 @@ public class KitRequestShipping extends KitRequest { private String labelUrlReturn; @ColumnName (DBConstants.DSM_TRACKING_TO) - private String trackingNumberTo; + private String trackingToId; @ColumnName (DBConstants.DSM_TRACKING_RETURN) private String trackingReturnId; @@ -268,9 +267,9 @@ public KitRequestShipping(String collaboratorParticipantId, String kitTypeName, upsTrackingStatus, upsReturnStatus, externalOrderDate, careEvolve, uploadReason, null, null, null); } - public KitRequestShipping(String participantId, String collaboratorParticipantId, String dsmKitId, String realm, String trackingNumberTo, String receiveDateString, String hruid, String gender) { + public KitRequestShipping(String participantId, String collaboratorParticipantId, String dsmKitId, String realm, String trackingToId, String receiveDateString, String hruid, String gender) { this(participantId, collaboratorParticipantId, null, null, realm, null, null, null, null, null, - trackingNumberTo, null, null, null, null, null, null, null, + trackingToId, null, null, null, null, null, null, null, null, null, null, dsmKitId, null, null, null, null, null, null, null, null, null, null, null, null, null, null, receiveDateString, hruid, gender); } @@ -288,7 +287,7 @@ public KitRequestShipping(Long dsmKitRequestId, Long dsmKitId, String easypostTo // shippingId = ddp_label !!! public KitRequestShipping(String participantId, String collaboratorParticipantId, String bspCollaboratorSampleId, String shippingId, String realm, String kitTypeName, Long dsmKitRequestId, Long dsmKitId, String labelUrlTo, String labelUrlReturn, - String trackingNumberTo, String trackingReturnId, + String trackingToId, String trackingReturnId, String easypostTrackingToUrl, String trackingUrlReturn, Long scanDate, Boolean error, String message, Long receiveDate, String easypostAddressId, Long deactivatedDate, String deactivationReason, String kitLabel, Boolean express, String easypostToId, Long labelDate, String easypostShipmentStatus, @@ -303,7 +302,7 @@ public KitRequestShipping(String participantId, String collaboratorParticipantId this.dsmKitId = dsmKitId; this.labelUrlTo = labelUrlTo; this.labelUrlReturn = labelUrlReturn; - this.trackingNumberTo = trackingNumberTo; + this.trackingToId = trackingToId; this.trackingReturnId = trackingReturnId; this.easypostTrackingToUrl = easypostTrackingToUrl; this.easypostTrackingReturnUrl = trackingUrlReturn; @@ -377,7 +376,7 @@ public static KitRequestShipping getKitRequestShipping(@NonNull ResultSet rs) th ); if (DBUtil.columnExists(rs, DBConstants.UPS_STATUS_DESCRIPTION) && StringUtils.isNotBlank(rs.getString(DBConstants.UPS_STATUS_DESCRIPTION))) { String upsPackageTrackingNumber = rs.getString(DBConstants.UPS_PACKAGE_TABLE_ABBR + DBConstants.UPS_TRACKING_NUMBER); - if (StringUtils.isNotBlank(upsPackageTrackingNumber) && upsPackageTrackingNumber.equals(kitRequestShipping.getTrackingNumberTo())) { + if (StringUtils.isNotBlank(upsPackageTrackingNumber) && upsPackageTrackingNumber.equals(kitRequestShipping.getTrackingToId())) { kitRequestShipping.setUpsTrackingStatus(rs.getString(DBConstants.UPS_STATUS_DESCRIPTION)); } @@ -1031,7 +1030,7 @@ public static void updateKit(String dsmKitId, Shipment participantShipment, Ship stmt.setString(7, participantTracker.getPublicUrl()); kitRequestShipping.setLabelUrlTo(participantLabel.getLabelUrl()); kitRequestShipping.setEasypostToId(participantShipment.getId()); - kitRequestShipping.setTrackingNumberTo(participantShipment.getTrackingCode()); + kitRequestShipping.setTrackingToId(participantShipment.getTrackingCode()); kitRequestShipping.setEasypostTrackingToUrl(participantTracker.getPublicUrl()); //TODO } else { diff --git a/src/main/java/org/broadinstitute/dsm/jobs/ExternalShipperJob.java b/src/main/java/org/broadinstitute/dsm/jobs/ExternalShipperJob.java index 3b77c5fa0..f6f895ab8 100644 --- a/src/main/java/org/broadinstitute/dsm/jobs/ExternalShipperJob.java +++ b/src/main/java/org/broadinstitute/dsm/jobs/ExternalShipperJob.java @@ -26,7 +26,7 @@ public void execute(JobExecutionContext context) { shipper.updateOrderStatusForPendingKitRequests(kitType.getInstanceId()); Instant now = Instant.now(); Instant dynamicStartTime = now.minus(5, ChronoUnit.DAYS); - shipper.orderConfirmation(dynamicStartTime.toEpochMilli(), now.toEpochMilli()); + shipper.orderConfirmation(dynamicStartTime.toEpochMilli(), now.toEpochMilli(), kitType.getInstanceId()); } catch (Exception e) { throw new RuntimeException("Failed to get status and/or confirmation ", e); diff --git a/src/main/java/org/broadinstitute/dsm/model/KitRequestExternal.java b/src/main/java/org/broadinstitute/dsm/model/KitRequestExternal.java index 76ea7d3c2..8f1961701 100644 --- a/src/main/java/org/broadinstitute/dsm/model/KitRequestExternal.java +++ b/src/main/java/org/broadinstitute/dsm/model/KitRequestExternal.java @@ -78,7 +78,7 @@ public static void updateKitRequestResponse(@NonNull Connection conn, String ext // update kit request with response of external shipper public static void updateKitRequestResponse(@NonNull Connection conn, String trackingIdTo, String trackingIdReturn, String kitLabel, long sentDate, - String sentBy, String dsmKitRequestId) { + String sentBy, String dsmKitRequestId, int ddpInstanceId) { try (PreparedStatement stmt = conn.prepareStatement(SQL_UPDATE_KIT_EXTERNAL_SHIPPER)) { stmt.setString(1, trackingIdTo); stmt.setString(2, trackingIdReturn); @@ -97,9 +97,15 @@ public static void updateKitRequestResponse(@NonNull Connection conn, String tra } KitRequestShipping kitRequestShipping = new KitRequestShipping(); kitRequestShipping.setDsmKitRequestId(Long.valueOf(dsmKitRequestId)); - kitRequestShipping.setTrac(Long.valueOf(dsmKitRequestId)); - kitRequestShipping.setDsmKitRequestId(Long.valueOf(dsmKitRequestId)); - kitRequestShipping.setDsmKitRequestId(Long.valueOf(dsmKitRequestId)); + kitRequestShipping.setTrackingToId(trackingIdTo); + kitRequestShipping.setTrackingReturnId(trackingIdReturn); + kitRequestShipping.setKitLabel(kitLabel); + kitRequestShipping.setScanDate(sentDate); + + DDPInstanceDto ddpInstanceDto = new DDPInstanceDao().getDDPInstanceByInstanceId(ddpInstanceId).orElseThrow(); + + UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, ESObjectConstants.DSM_KIT_REQUEST_ID, ESObjectConstants.DSM_KIT_REQUEST_ID, dsmKitRequestId) + .export(); } } diff --git a/src/main/java/org/broadinstitute/dsm/util/externalShipper/ExternalShipper.java b/src/main/java/org/broadinstitute/dsm/util/externalShipper/ExternalShipper.java index 6e608d412..9e95e35af 100644 --- a/src/main/java/org/broadinstitute/dsm/util/externalShipper/ExternalShipper.java +++ b/src/main/java/org/broadinstitute/dsm/util/externalShipper/ExternalShipper.java @@ -1,5 +1,6 @@ package org.broadinstitute.dsm.util.externalShipper; +import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; import org.broadinstitute.dsm.model.KitRequest; import org.broadinstitute.dsm.model.KitRequestSettings; import org.broadinstitute.dsm.util.EasyPostUtil; @@ -18,7 +19,7 @@ public interface ExternalShipper { */ void updateOrderStatusForPendingKitRequests(int instanceId); - public void orderConfirmation(long startDate, long endDate) throws Exception; + void orderConfirmation(long startDate, long endDate, int ddpInstanceId) throws Exception; public void orderCancellation(ArrayList kitRequests) throws Exception; diff --git a/src/main/java/org/broadinstitute/dsm/util/externalShipper/GBFRequestUtil.java b/src/main/java/org/broadinstitute/dsm/util/externalShipper/GBFRequestUtil.java index 8010a4574..49b4fdfe2 100644 --- a/src/main/java/org/broadinstitute/dsm/util/externalShipper/GBFRequestUtil.java +++ b/src/main/java/org/broadinstitute/dsm/util/externalShipper/GBFRequestUtil.java @@ -297,7 +297,7 @@ else if (status.getOrderStatus().contains("CANCELLED") && (StringUtils.isBlank(k } } - public static void processingSingleConfirmation(Response gbfResponse, ShippingConfirmation confirmation) throws Exception { + public static void processingSingleConfirmation(Response gbfResponse, ShippingConfirmation confirmation, int ddpInstanceId) throws Exception { logger.info("Got confirmation for " + confirmation.getOrderNumber()); Node node = GBFRequestUtil.getXMLNode(gbfResponse.getXML(), XML_NODE_EXPRESSION.replace("%1", confirmation.getOrderNumber())); String externalResponse = getStringFromNode(node); @@ -318,7 +318,7 @@ public static void processingSingleConfirmation(Response gbfResponse, ShippingCo } KitRequestExternal.updateKitRequestResponse(conn, confirmation.getTracking(), item.getReturnTracking(), kitLabel, SystemUtil.getLongFromDateString(confirmation.getShipDate()), EXTERNAL_SHIPPER_NAME, - dsmKitRequestId); + dsmKitRequestId, ddpInstanceId); counter++; logger.info("Updated confirmation information for : " + dsmKitRequestId + " " + kitLabel); } @@ -334,7 +334,7 @@ public static void processingSingleConfirmation(Response gbfResponse, ShippingCo // The confirmation, dependent upon level of detail required, is a shipping receipt to prove completion. // Confirmation may include order number, client(participant) ID, outbound tracking number, return tracking number(s), line item(s), kit serial number(s), etc. - public void orderConfirmation(long startDate, long endDate) throws Exception { + public void orderConfirmation(long startDate, long endDate, int ddpInstanceId) throws Exception { JSONObject payload = new JSONObject().put("startDate", SystemUtil.getDateFormatted(startDate)).put("endDate", SystemUtil.getDateFormatted(endDate)); String sendRequest = DSMServer.getBaseUrl(getExternalShipperName()) + CONFIRM_ENDPOINT; logger.info("payload: " + payload.toString()); @@ -350,7 +350,7 @@ public void orderConfirmation(long startDate, long endDate) throws Exception { logger.info("Number of confirmations received: " + confirmationList.size()); for (ShippingConfirmation confirmation : confirmationList) { try { - processingSingleConfirmation(gbfResponse, confirmation); + processingSingleConfirmation(gbfResponse, confirmation, ddpInstanceId); } catch (Exception e) { logger.error("Could not process confirmation for " + confirmation.getOrderNumber(), e); diff --git a/src/test/java/org/broadinstitute/dsm/GBFTest.java b/src/test/java/org/broadinstitute/dsm/GBFTest.java index 6862f09ad..f6eea78da 100644 --- a/src/test/java/org/broadinstitute/dsm/GBFTest.java +++ b/src/test/java/org/broadinstitute/dsm/GBFTest.java @@ -232,7 +232,7 @@ public void inputConfirmationGBFKit() throws Exception { for (ShippingConfirmation confirmation : confirmationList) { if (kits.containsKey(confirmation.getOrderNumber())) { try { - gbf.processingSingleConfirmation(gbfResponse, confirmation); + gbf.processingSingleConfirmation(gbfResponse, confirmation, 0); } catch (Exception e) { logger.error("Could not process confirmation for " + confirmation.getOrderNumber(), e); From 6e9e7e524e75bd4422ad48a1ed8e38bdf3b556a8 Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 10 Feb 2022 10:06:20 +0400 Subject: [PATCH 399/405] DDP-7443 | call export() in "updateKitRequest" --- .../java/org/broadinstitute/dsm/model/KitRequestExternal.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/KitRequestExternal.java b/src/main/java/org/broadinstitute/dsm/model/KitRequestExternal.java index 8f1961701..c68f801fb 100644 --- a/src/main/java/org/broadinstitute/dsm/model/KitRequestExternal.java +++ b/src/main/java/org/broadinstitute/dsm/model/KitRequestExternal.java @@ -57,7 +57,7 @@ public static void updateKitRequest(Connection conn, String externalOrderStatus, kitRequestShipping.setExternalOrderDate(externalOrderDate); DDPInstanceDto ddpInstanceDto = new DDPInstanceDao().getDDPInstanceByInstanceId(instanceId).orElseThrow(); UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, ESObjectConstants.DSM_KIT_REQUEST_ID, - ESObjectConstants.DSM_KIT_REQUEST_ID, dsmKitRequestId); + ESObjectConstants.DSM_KIT_REQUEST_ID, dsmKitRequestId).export(); } // update kit request with response of external shipper From 7bdfb41aa926152d642a75639d869698d179b84e Mon Sep 17 00:00:00 2001 From: Nika Date: Thu, 10 Feb 2022 12:27:58 +0400 Subject: [PATCH 400/405] DDP-7443 | add export to ES in PubSubLookup.java --- .../broadinstitute/dsm/jobs/PubSubLookUp.java | 20 +++++++++++-- .../dsm/model/birch/DSMTestResult.java | 6 ++++ .../export/painless/ParamsGeneratorTest.java | 29 +++++++++++++++++++ 3 files changed, 52 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/jobs/PubSubLookUp.java b/src/main/java/org/broadinstitute/dsm/jobs/PubSubLookUp.java index 07f24ddec..550c94f8a 100644 --- a/src/main/java/org/broadinstitute/dsm/jobs/PubSubLookUp.java +++ b/src/main/java/org/broadinstitute/dsm/jobs/PubSubLookUp.java @@ -6,10 +6,14 @@ import org.apache.commons.lang3.StringUtils; import org.broadinstitute.ddp.db.SimpleResult; import org.broadinstitute.dsm.db.DDPInstance; +import org.broadinstitute.dsm.db.KitRequestShipping; +import org.broadinstitute.dsm.db.dto.ddp.instance.DDPInstanceDto; import org.broadinstitute.dsm.model.KitDDPNotification; import org.broadinstitute.dsm.model.birch.DSMTestResult; import org.broadinstitute.dsm.model.birch.TestBostonResult; +import org.broadinstitute.dsm.model.elastic.export.painless.UpsertPainlessFacade; import org.broadinstitute.dsm.statics.DBConstants; +import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.EventUtil; import org.broadinstitute.dsm.util.NotificationUtil; import org.slf4j.Logger; @@ -39,7 +43,7 @@ public static void processCovidTestResults(Connection conn, PubsubMessage messag if(ddpInstance.isHasRole()) { logger.info("Processing test results for " + testBostonResult.getSampleId()); if (shouldWriteResultIntoDB(testBostonResult)) { - writeResultsIntoDB(testBostonResult); + writeResultsIntoDB(testBostonResult, ddpInstance); tellPepperAboutTheNewResults(conn, testBostonResult);// notify pepper if we update DB notifyPIs(ddpInstance, testBostonResult, notificationUtil); } @@ -139,7 +143,7 @@ private static DSMTestResult[] getLatestKitTestResults(TestBostonResult testBost return dsmTestResult; } - public static void writeResultsIntoDB(TestBostonResult testBostonResult) { + public static void writeResultsIntoDB(TestBostonResult testBostonResult, DDPInstance ddpInstance) { DSMTestResult[] dsmTestResultArray = getLatestKitTestResults(testBostonResult); DSMTestResult[] array = null; DSMTestResult newDsmTestResult = new DSMTestResult(testBostonResult.getResult(), testBostonResult.getTimeCompleted(), testBostonResult.isCorrected()); @@ -179,8 +183,18 @@ public static void writeResultsIntoDB(TestBostonResult testBostonResult) { throw new RuntimeException("Couldn't update the test results for kit label" + testBostonResult.getSampleId(), results.resultException); } else { + KitRequestShipping kitRequestShipping = new KitRequestShipping(); + kitRequestShipping.setTestResult(finalArray); + + DDPInstanceDto ddpInstanceDto = new DDPInstanceDto.Builder() + .withEsParticipantIndex(ddpInstance.getParticipantIndexES()) + .withInstanceName(ddpInstance.getName()) + .build(); + + UpsertPainlessFacade.of(DBConstants.DDP_KIT_REQUEST_ALIAS, kitRequestShipping, ddpInstanceDto, ESObjectConstants.KIT_LABEL, ESObjectConstants.KIT_LABEL, testBostonResult.getSampleId()) + .export(); + logger.info("Updated test result for kit with external id " + testBostonResult.getSampleId()); } - } } diff --git a/src/main/java/org/broadinstitute/dsm/model/birch/DSMTestResult.java b/src/main/java/org/broadinstitute/dsm/model/birch/DSMTestResult.java index 10df6a5ed..c7a76b27c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/birch/DSMTestResult.java +++ b/src/main/java/org/broadinstitute/dsm/model/birch/DSMTestResult.java @@ -1,5 +1,6 @@ package org.broadinstitute.dsm.model.birch; +import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; @Data @@ -13,4 +14,9 @@ public DSMTestResult(String result, String timeCompleted, boolean isCorrected){ this.timeCompleted = timeCompleted; this.isCorrected = isCorrected; } + + @JsonProperty("isCorrected") + public boolean isCorrected() { + return isCorrected; + } } diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGeneratorTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGeneratorTest.java index 6769b7af7..75f5661f1 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGeneratorTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/export/painless/ParamsGeneratorTest.java @@ -1,9 +1,17 @@ package org.broadinstitute.dsm.model.elastic.export.painless; +import com.google.gson.Gson; import org.broadinstitute.dsm.db.KitRequestShipping; +import org.broadinstitute.dsm.model.birch.DSMTestResult; +import org.broadinstitute.dsm.model.birch.TestBostonResult; +import org.broadinstitute.dsm.statics.DBConstants; +import org.broadinstitute.dsm.statics.ESObjectConstants; +import org.broadinstitute.dsm.util.proxy.jackson.ObjectMapperSingleton; import org.junit.Assert; import org.junit.Test; +import java.io.Serializable; +import java.util.List; import java.util.Map; public class ParamsGeneratorTest { @@ -22,4 +30,25 @@ public void generate() { Assert.assertEquals("msg", kitRequestShippingObj.get("message")); } + @Test + public void generateKitRequest() { + + DSMTestResult[] array = new DSMTestResult[] { new DSMTestResult("Negative", "2012-05-02", true) }; + String finalArray = ObjectMapperSingleton.writeValueAsString(array); + + KitRequestShipping kitRequestShipping = new KitRequestShipping(); + kitRequestShipping.setTestResult(finalArray); + + ParamsGenerator paramsGenerator = new ParamsGenerator(kitRequestShipping, null); + Map actual = paramsGenerator.generate(); + + Map expected = Map.of(ESObjectConstants.DSM, Map.of(ESObjectConstants.KIT_REQUEST_SHIPPING, + Map.of(ESObjectConstants.KIT_TEST_RESULT, List.of( + Map.of("result", "Negative", "timeCompleted", "2012-05-02", "isCorrected", true))) + )); + + Assert.assertEquals(expected, actual); + + } + } \ No newline at end of file From 4f894f406b725efb1f31fa90e1c3f5255276f56e Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 11 Feb 2022 10:21:23 +0400 Subject: [PATCH 401/405] DDP-7443 | fix getting testResult in sourcemapdeserialzier --- .../dsm/model/elastic/search/SourceMapDeserializer.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java b/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java index b6bc6c492..1edf70000 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializer.java @@ -84,8 +84,7 @@ String getDynamicFieldsValueAsJson(Map clonedMap) { } String convertTestResultValueAsJson(Map clonedMap) { - Map testResult = (Map) clonedMap.get(ESObjectConstants.KIT_TEST_RESULT); - return ObjectMapperSingleton.writeValueAsString(List.of(testResult)); + return ObjectMapperSingleton.writeValueAsString(clonedMap.get(ESObjectConstants.KIT_TEST_RESULT)); } protected Map convertDynamicFieldsFromCamelCaseToPascalCase(Map dynamicFields) { From d0a6e9cc26c05a7f800a1461824419053cb46976 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 11 Feb 2022 10:50:11 +0400 Subject: [PATCH 402/405] DDP-7443 | change in TestREquestCollectionQueryBuilder to construct propert nested path --- .../query/TestResultCollectionQueryBuilder.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/TestResultCollectionQueryBuilder.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/TestResultCollectionQueryBuilder.java index 01779a844..ffef5e4ca 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/TestResultCollectionQueryBuilder.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/query/TestResultCollectionQueryBuilder.java @@ -1,17 +1,17 @@ package org.broadinstitute.dsm.model.elastic.filter.query; -import org.broadinstitute.dsm.model.elastic.filter.Operator; -import org.broadinstitute.dsm.model.elastic.filter.splitter.BaseSplitter; +import org.apache.lucene.search.join.ScoreMode; import org.broadinstitute.dsm.statics.DBConstants; +import org.elasticsearch.index.query.NestedQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; -public class TestResultCollectionQueryBuilder extends CollectionQueryBuilder { +public class TestResultCollectionQueryBuilder extends BaseQueryBuilder { public static final String TEST_RESULT = "testResult"; @Override - public QueryBuilder buildEachQuery(Operator operator, QueryPayload queryPayload, BaseSplitter splitter) { - queryPayload.setPath(String.join(DBConstants.ALIAS_DELIMITER, queryPayload.getPath(), splitter.getFieldName())); - return super.buildEachQuery(operator, queryPayload, splitter); + protected QueryBuilder build(QueryBuilder queryBuilder) { + String path = String.join(DBConstants.ALIAS_DELIMITER,this.payload.getPath(), splitter.getFieldName()); + return new NestedQueryBuilder(path, queryBuilder, ScoreMode.Avg); } } From adc2351172b4542036858858c8e0d8ff7934288f Mon Sep 17 00:00:00 2001 From: Nika Date: Fri, 11 Feb 2022 11:15:41 +0400 Subject: [PATCH 403/405] DDP-7443 | fix gedFieldWithAlias() --- .../filter/splitter/JsonContainsSplitter.java | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitter.java b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitter.java index 32f503f1b..845aa789c 100644 --- a/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitter.java +++ b/src/main/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitter.java @@ -7,34 +7,37 @@ import org.broadinstitute.dsm.statics.ESObjectConstants; import org.broadinstitute.dsm.util.ElasticSearchUtil; +import java.util.Arrays; + public class JsonContainsSplitter extends BaseSplitter { + public static final int ALIAS_INDEX = 0; + public static final int FIELD_INDEX = 1; + @Override public String[] split() { - return filter.split(Filter.JSON_CONTAINS)[1] + return filter.split(Filter.JSON_CONTAINS)[FIELD_INDEX] .replace(Filter.OPEN_PARENTHESIS, StringUtils.EMPTY) - .replace(Filter.CLOSE_PARENTHESIS,StringUtils.EMPTY) + .replace(Filter.CLOSE_PARENTHESIS, StringUtils.EMPTY) .split(Filter.JSON_OBJECT); } @Override public String[] getValue() { - return new String[] {splittedFilter[1].split(Util.COMMA_SEPARATOR)[1].trim()}; - } - - @Override - public String getInnerProperty() { - return splittedFilter[1] - .split(Util.COMMA_SEPARATOR)[0] - .replaceAll(Filter.SINGLE_QUOTE, StringUtils.EMPTY); + return new String[]{splittedFilter[FIELD_INDEX].split(Util.COMMA_SEPARATOR)[FIELD_INDEX].trim()}; } @Override protected String[] getFieldWithAlias() { - return splittedFilter[0] + String[] fieldWithAlias = splittedFilter[ALIAS_INDEX] .replace(Util.COMMA_SEPARATOR, StringUtils.EMPTY) .trim() .split(ElasticSearchUtil.ESCAPE_CHARACTER_DOT_SEPARATOR); + String innerProperty = splittedFilter[FIELD_INDEX] + .replace(Filter.SINGLE_QUOTE, StringUtils.EMPTY) + .trim() + .split(Util.COMMA_SEPARATOR)[ALIAS_INDEX].trim(); + return new String[]{fieldWithAlias[ALIAS_INDEX], fieldWithAlias[FIELD_INDEX], innerProperty}; } } From 766825b2d92f9c2b34c6fab58c4f29ff12bf5fc3 Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 11 Feb 2022 12:54:05 +0400 Subject: [PATCH 404/405] DDP-7443 | add kitLabel for painless script --- src/main/java/org/broadinstitute/dsm/jobs/PubSubLookUp.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/broadinstitute/dsm/jobs/PubSubLookUp.java b/src/main/java/org/broadinstitute/dsm/jobs/PubSubLookUp.java index 550c94f8a..1affbd39d 100644 --- a/src/main/java/org/broadinstitute/dsm/jobs/PubSubLookUp.java +++ b/src/main/java/org/broadinstitute/dsm/jobs/PubSubLookUp.java @@ -185,6 +185,7 @@ public static void writeResultsIntoDB(TestBostonResult testBostonResult, DDPInst else { KitRequestShipping kitRequestShipping = new KitRequestShipping(); kitRequestShipping.setTestResult(finalArray); + kitRequestShipping.setKitLabel(testBostonResult.getSampleId()); DDPInstanceDto ddpInstanceDto = new DDPInstanceDto.Builder() .withEsParticipantIndex(ddpInstance.getParticipantIndexES()) From 6572efce44ac325f43ef94e1b7821439dbb9cabb Mon Sep 17 00:00:00 2001 From: gmakhara Date: Fri, 11 Feb 2022 16:20:38 +0400 Subject: [PATCH 405/405] DDP-7443 | update tests according to the changes --- .../filter/query/TestResultCollectionQueryBuilderTest.java | 2 +- .../model/elastic/filter/splitter/JsonContainsSplitterTest.java | 2 +- .../dsm/model/elastic/search/SourceMapDeserializerTest.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/TestResultCollectionQueryBuilderTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/TestResultCollectionQueryBuilderTest.java index 20a3413a8..35ffd3c1d 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/TestResultCollectionQueryBuilderTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/query/TestResultCollectionQueryBuilderTest.java @@ -17,7 +17,7 @@ public class TestResultCollectionQueryBuilderTest { public void build() { BaseQueryBuilder queryBuilder = new TestResultCollectionQueryBuilder(); QueryPayload queryPayload = - new QueryPayload("dsm.kitRequestShipping", "isCorrected", new FilterParser().parse(new String[] {"'true'"})); + new QueryPayload("dsm.kitRequestShipping", "testResult.isCorrected", new FilterParser().parse(new String[] {"'true'"})); String filter = "JSON_CONTAINS(k.test_result, JSON_OBJECT('isCorrected', 'true'))"; JsonContainsSplitter splitter = new JsonContainsSplitter(); splitter.setFilter(filter); diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitterTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitterTest.java index 567d89982..9164f46f7 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitterTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/filter/splitter/JsonContainsSplitterTest.java @@ -21,7 +21,7 @@ public void getInnerProperty() { BaseSplitter splitter = new JsonContainsSplitter(); String filter = "JSON_CONTAINS(k.test_result, JSON_OBJECT('isCorrected', 'true'))"; splitter.setFilter(filter); - assertEquals("isCorrected", splitter.getInnerProperty()); + assertEquals("testResult.isCorrected", splitter.getInnerProperty()); } @Test diff --git a/src/test/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializerTest.java b/src/test/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializerTest.java index 3a96ac647..454a6733a 100644 --- a/src/test/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializerTest.java +++ b/src/test/java/org/broadinstitute/dsm/model/elastic/search/SourceMapDeserializerTest.java @@ -45,7 +45,7 @@ public void testResultToJson() { testResultInner.put("timeCompleted", "2020-01-01"); testResultInner.put("result", "Positive"); - Map testResult = Map.of("testResult", testResultInner); + Map testResult = Map.of("testResult", List.of(testResultInner)); String testResultValueAsJson = sourceMapDeserializer.convertTestResultValueAsJson(testResult);