Skip to content

Commit

Permalink
Merge pull request #74 from uc-cdis/fix/order_of_variables_in_attriti…
Browse files Browse the repository at this point in the history 
…on_csv

Feat: adjust concept.RetrieveAttritionTable to produce rows in new order
  • Loading branch information
@pieterlukasse
pieterlukasse authored Jun 29, 2023
2 parents 012dc01 + 743c8aa commit 5d0409d
Show file tree
Hide file tree
Showing 4 changed files with 138 additions and 138 deletions.
114 changes: 44 additions & 70 deletions controllers/concept.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,60 +167,8 @@ func generateRowForVariable(variableName string, breakdownConceptValuesToPeopleC
return row
}

func (u ConceptController) GetConceptVariablesAttritionRows(sourceId int, cohortId int, conceptIds []int64, breakdownConceptId int64, sortedConceptValues []string) ([][]string, error) {
conceptIdToName := make(map[int64]string)
conceptInformations, err := u.conceptModel.RetrieveInfoBySourceIdAndConceptIds(sourceId, conceptIds)
if err != nil {
return nil, fmt.Errorf("could not retrieve concept informations due to error: %s", err.Error())
}
for _, conceptInformation := range conceptInformations {
conceptIdToName[conceptInformation.ConceptId] = conceptInformation.ConceptName
}

var rows [][]string
for idx, conceptId := range conceptIds {
// run each query with a longer list of filterConceptIds, until the last query is run with them all:
filterConceptIds := conceptIds[0 : idx+1]
// use empty cohort pairs list:
filterCohortPairs := []utils.CustomDichotomousVariableDef{}
breakdownStats, err := u.conceptModel.RetrieveBreakdownStatsBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId, cohortId, filterConceptIds, filterCohortPairs, breakdownConceptId)
if err != nil {
return nil, fmt.Errorf("could not retrieve concept Breakdown for concepts %v due to error: %s", filterConceptIds, err.Error())
}

conceptValuesToPeopleCount := getConceptValueToPeopleCount(breakdownStats)
variableName := conceptIdToName[conceptId]
log.Printf("Generating row for variable with name %s", variableName)
generatedRow := generateRowForVariable(variableName, conceptValuesToPeopleCount, sortedConceptValues)
rows = append(rows, generatedRow)
}

return rows, nil
}

func (u ConceptController) GetCustomDichotomousVariablesAttritionRows(sourceId int, cohortId int, filterConceptIds []int64, filterCohortPairs []utils.CustomDichotomousVariableDef, breakdownConceptId int64, sortedConceptValues []string) ([][]string, error) {
// TODO - this function is very similar to GetConceptVariablesAttritionRows above and they can probably be merged.
var rows [][]string
for idx, cohortPair := range filterCohortPairs {
// run each query with the full list of filterConceptIds and an increasingly longer list of filterCohortPairs, until the last query is run with them all:
filterCohortPairs := filterCohortPairs[0 : idx+1]
breakdownStats, err := u.conceptModel.RetrieveBreakdownStatsBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId, cohortId, filterConceptIds, filterCohortPairs, breakdownConceptId)
if err != nil {
return nil, fmt.Errorf("could not retrieve concept Breakdown for dichotomous variables %v due to error: %s", filterConceptIds, err.Error())
}

conceptValuesToPeopleCount := getConceptValueToPeopleCount(breakdownStats)
variableName := cohortPair.ProvidedName
log.Printf("Generating row for variable...")
generatedRow := generateRowForVariable(variableName, conceptValuesToPeopleCount, sortedConceptValues)
rows = append(rows, generatedRow)
}

return rows, nil
}

func (u ConceptController) RetrieveAttritionTable(c *gin.Context) {
sourceId, cohortId, conceptIds, cohortPairs, err := utils.ParseSourceIdAndCohortIdAndVariablesList(c)
sourceId, cohortId, conceptIdsAndCohortPairs, err := utils.ParseSourceIdAndCohortIdAndVariablesAsSingleList(c)
if err != nil {
log.Printf("Error: %s", err.Error())
c.JSON(http.StatusBadRequest, gin.H{"message": "bad request", "error": err.Error()})
Expand All @@ -245,7 +193,7 @@ func (u ConceptController) RetrieveAttritionTable(c *gin.Context) {
breakdownStats, err := u.conceptModel.RetrieveBreakdownStatsBySourceIdAndCohortId(sourceId, cohortId, breakdownConceptId)
if err != nil {
log.Printf("Error: %s", err.Error())
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error retrieving concept breakdown with filtered conceptIds", "error": err.Error()})
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error retrieving concept breakdown for given cohortId", "error": err.Error()})
c.Abort()
return
}
Expand All @@ -255,32 +203,58 @@ func (u ConceptController) RetrieveAttritionTable(c *gin.Context) {
headerAndNonFilteredRow, err := u.GenerateHeaderAndNonFilteredRow(breakdownStats, sortedConceptValues, cohortName)
if err != nil {
log.Printf("Error: %s", err.Error())
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error retrieving concept breakdown with filtered conceptIds", "error": err.Error()})
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error generating concept breakdown header and cohort rows", "error": err.Error()})
c.Abort()
return
}

// append concepts to attrition table:
conceptVariablesAttritionRows, err := u.GetConceptVariablesAttritionRows(sourceId, cohortId, conceptIds, breakdownConceptId, sortedConceptValues)
otherAttritionRows, err := u.GetAttritionRowForConceptIdsAndCohortPairs(sourceId, cohortId, conceptIdsAndCohortPairs, breakdownConceptId, sortedConceptValues)
if err != nil {
log.Printf("Error: %s", err.Error())
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error retrieving concept breakdown with filtered conceptIds", "error": err.Error()})
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error retrieving concept breakdown rows for filter conceptIds and cohortPairs", "error": err.Error()})
c.Abort()
return
}
// append custom dichotomous items to attrition table:
customDichotomousVariablesAttritionRows, err := u.GetCustomDichotomousVariablesAttritionRows(sourceId, cohortId, conceptIds, cohortPairs, breakdownConceptId, sortedConceptValues)
if err != nil {
log.Printf("Error: %s", err.Error())
c.JSON(http.StatusInternalServerError, gin.H{"message": "Error retrieving concept breakdown with custom dichotomous variables (aka cohortpairs)", "error": err.Error()})
c.Abort()
return
b := GenerateAttritionCSV(headerAndNonFilteredRow, otherAttritionRows)
c.String(http.StatusOK, b.String())
}

func (u ConceptController) GetAttritionRowForConceptIdsAndCohortPairs(sourceId int, cohortId int, conceptIdsAndCohortPairs []interface{}, breakdownConceptId int64, sortedConceptValues []string) ([][]string, error) {
var otherAttritionRows [][]string
for idx, conceptIdOrCohortPair := range conceptIdsAndCohortPairs {
// attrition filter: run each query with an increasingly longer list of filterConceptIdsAndCohortPairs, until the last query is run with them all:
filterConceptIdsAndCohortPairs := conceptIdsAndCohortPairs[0 : idx+1]

attritionRow, err := u.GetAttritionRowForConceptIdOrCohortPair(sourceId, cohortId, conceptIdOrCohortPair, filterConceptIdsAndCohortPairs, breakdownConceptId, sortedConceptValues)
if err != nil {
log.Printf("Error: %s", err.Error())
return nil, err
}
otherAttritionRows = append(otherAttritionRows, attritionRow)
}
return otherAttritionRows, nil
}

// concat all rows:
var allVariablesAttritionRows = append(conceptVariablesAttritionRows, customDichotomousVariablesAttritionRows...)
b := GenerateAttritionCSV(headerAndNonFilteredRow, allVariablesAttritionRows)
c.String(http.StatusOK, b.String())
func (u ConceptController) GetAttritionRowForConceptIdOrCohortPair(sourceId int, cohortId int, conceptIdOrCohortPair interface{}, filterConceptIdsAndCohortPairs []interface{}, breakdownConceptId int64, sortedConceptValues []string) ([]string, error) {
filterConceptIds, filterCohortPairs := utils.GetConceptIdsAndCohortPairsAsSeparateLists(filterConceptIdsAndCohortPairs)
breakdownStats, err := u.conceptModel.RetrieveBreakdownStatsBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId, cohortId, filterConceptIds, filterCohortPairs, breakdownConceptId)
if err != nil {
return nil, fmt.Errorf("could not retrieve concept Breakdown for concepts %v dichotomous variables %v due to error: %s", filterConceptIds, filterCohortPairs, err.Error())
}
conceptValuesToPeopleCount := getConceptValueToPeopleCount(breakdownStats)
variableName := ""
switch convertedItem := conceptIdOrCohortPair.(type) {
case int64:
conceptInformation, err := u.conceptModel.RetrieveInfoBySourceIdAndConceptId(sourceId, convertedItem)
if err != nil {
return nil, fmt.Errorf("could not retrieve concept details for %v due to error: %s", convertedItem, err.Error())
}
variableName = conceptInformation.ConceptName
case utils.CustomDichotomousVariableDef:
variableName = convertedItem.ProvidedName
}
log.Printf("Generating row for variable with name %s", variableName)
generatedRow := generateRowForVariable(variableName, conceptValuesToPeopleCount, sortedConceptValues)
return generatedRow, nil
}

func getSortedConceptValues(breakdownStats []*models.ConceptBreakdown) []string {
Expand Down
1 change: 1 addition & 0 deletions models/concept.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (

type ConceptI interface {
RetriveAllBySourceId(sourceId int) ([]*Concept, error)
RetrieveInfoBySourceIdAndConceptId(sourceId int, conceptId int64) (*ConceptSimple, error)
RetrieveInfoBySourceIdAndConceptIds(sourceId int, conceptIds []int64) ([]*ConceptSimple, error)
RetrieveInfoBySourceIdAndConceptTypes(sourceId int, conceptTypes []string) ([]*ConceptSimple, error)
RetrieveBreakdownStatsBySourceIdAndCohortId(sourceId int, cohortDefinitionId int, breakdownConceptId int64) ([]*ConceptBreakdown, error)
Expand Down
97 changes: 43 additions & 54 deletions tests/controllers_tests/controllers_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,6 @@ var cohortDefinitionControllerNeedsDb = controllers.NewCohortDefinitionControlle
// instance of the controller that talks to a mock implementation of the model:
var cohortDefinitionController = controllers.NewCohortDefinitionController(*new(dummyCohortDefinitionDataModel))

var conceptController = controllers.NewConceptController(*new(dummyConceptDataModel), *new(dummyCohortDefinitionDataModel))

type dummyCohortDataModel struct{}

func (h dummyCohortDataModel) RetrieveDataBySourceIdAndCohortIdAndConceptIdsOrderedByPersonId(sourceId int, cohortDefinitionId int, conceptIds []int64) ([]*models.PersonConceptAndValue, error) {
Expand Down Expand Up @@ -129,11 +127,28 @@ func (h dummyCohortDefinitionDataModel) GetAllCohortDefinitions() ([]*models.Coh
return nil, nil
}

var conceptController = controllers.NewConceptController(*new(dummyConceptDataModel), *new(dummyCohortDefinitionDataModel))

type dummyConceptDataModel struct{}

func (h dummyConceptDataModel) RetriveAllBySourceId(sourceId int) ([]*models.Concept, error) {
return nil, nil
}

func (h dummyConceptDataModel) RetrieveInfoBySourceIdAndConceptId(sourceId int, conceptId int64) (*models.ConceptSimple, error) {
conceptSimpleItems := []*models.ConceptSimple{
{ConceptId: 1234, ConceptName: "Concept A"},
{ConceptId: 5678, ConceptName: "Concept B"},
{ConceptId: 2090006880, ConceptName: "Concept C"},
}
for _, conceptSimple := range conceptSimpleItems {
if conceptSimple.ConceptId == conceptId {
return conceptSimple, nil
}
}
return nil, fmt.Errorf("concept id %d not found in mock data", conceptId)
}

func (h dummyConceptDataModel) RetrieveInfoBySourceIdAndConceptIds(sourceId int, conceptIds []int64) ([]*models.ConceptSimple, error) {
// dummy data with _some_ of the relevant fields:
conceptSimple := []*models.ConceptSimple{
Expand Down Expand Up @@ -169,8 +184,8 @@ func (h dummyConceptDataModel) RetrieveBreakdownStatsBySourceIdAndCohortId(sourc
}
func (h dummyConceptDataModel) RetrieveBreakdownStatsBySourceIdAndCohortIdAndConceptIdsAndCohortPairs(sourceId int, cohortDefinitionId int, filterConceptIds []int64, filterCohortPairs []utils.CustomDichotomousVariableDef, breakdownConceptId int64) ([]*models.ConceptBreakdown, error) {
conceptBreakdown := []*models.ConceptBreakdown{
{ConceptValue: "value1", NpersonsInCohortWithValue: 4},
{ConceptValue: "value2", NpersonsInCohortWithValue: 7},
{ConceptValue: "value1", NpersonsInCohortWithValue: 4 - len(filterCohortPairs)}, // simulate decreasing numbers as filter increases - the use of filterCohortPairs instead of filterConceptIds is otherwise meaningless here...
{ConceptValue: "value2", NpersonsInCohortWithValue: 7 - len(filterConceptIds)}, // simulate decreasing numbers as filter increases- the use of filterConceptIds instead of filterCohortPairs is otherwise meaningless here...
}
if dummyModelReturnError {
return nil, fmt.Errorf("error!")
Expand Down Expand Up @@ -603,75 +618,49 @@ func TestGenerateHeaderAndNonFilterRow(t *testing.T) {
}
}

func TestGetConceptVariablesAttritionRows(t *testing.T) {
func TestGetAttritionRowForConceptIdsAndCohortPairs(t *testing.T) {
setUp(t)
sourceId := 1
cohortId := 1
var breakdownConceptId int64 = 1
conceptIds := []int64{1234, 5678, 2090006880}
sortedConceptValues := []string{"value1", "value2"}

result, _ := conceptController.GetConceptVariablesAttritionRows(sourceId, cohortId, conceptIds, breakdownConceptId, sortedConceptValues)
if len(result) != 3 {
t.Errorf("Expected 3 data lines, found %d lines in total",
len(result))
t.Errorf("Lines: %s", result)
}

expectedLines := [][]string{
{"Concept A", "11", "4", "7"},
{"Concept B", "11", "4", "7"},
{"Concept C", "11", "4", "7"},
}

i := 0
for _, expectedLine := range expectedLines {
if !reflect.DeepEqual(expectedLine, result[i]) {
t.Errorf("header or non filter row line not as expected. \nExpected: \n%s \nFound: \n%s",
expectedLine, result[i])
}
i++
}
}
sortedConceptValues := []string{"value1", "value2", "value3"}

func TestGetCustomDichotomousVariablesAttritionRows(t *testing.T) {
setUp(t)
sourceId := 1
cohortId := 1
var breakdownConceptId int64 = 1
conceptIds := []int64{1234, 5678, 2090006880}
cohortPairs := []utils.CustomDichotomousVariableDef{
{
// mix of concept ids and CustomDichotomousVariableDef items:
conceptIdsAndCohortPairs := []interface{}{
int64(1234),
int64(5678),
utils.CustomDichotomousVariableDef{
CohortId1: 1,
CohortId2: 2,
ProvidedName: "testA12"},
{
int64(2090006880),
utils.CustomDichotomousVariableDef{
CohortId1: 3,
CohortId2: 4,
ProvidedName: "testB34"},
}

sortedConceptValues := []string{"value1", "value2", "value3"}

result, _ := conceptController.GetCustomDichotomousVariablesAttritionRows(sourceId, cohortId, conceptIds, cohortPairs, breakdownConceptId, sortedConceptValues)
if len(result) != 2 {
t.Errorf("Expected 3 data lines, found %d lines in total",
result, _ := conceptController.GetAttritionRowForConceptIdsAndCohortPairs(sourceId, cohortId, conceptIdsAndCohortPairs, breakdownConceptId, sortedConceptValues)
if len(result) != len(conceptIdsAndCohortPairs) {
t.Errorf("Expected %d data lines, found %d lines in total",
len(conceptIdsAndCohortPairs),
len(result))
t.Errorf("Lines: %s", result)
}

expectedLines := [][]string{
{"testA12", "11", "4", "7", "0"},
{"testB34", "11", "4", "7", "0"},
{"Concept A", "10", "4", "6", "0"},
{"Concept B", "9", "4", "5", "0"},
{"testA12", "8", "3", "5", "0"},
{"Concept C", "7", "3", "4", "0"},
{"testB34", "6", "2", "4", "0"},
}

i := 0
for _, expectedLine := range expectedLines {
for i, expectedLine := range expectedLines {
if !reflect.DeepEqual(expectedLine, result[i]) {
t.Errorf("header or non filter row line not as expected. \nExpected: \n%s \nFound: \n%s",
expectedLine, result[i])
}
i++
}
}

Expand Down Expand Up @@ -849,8 +838,8 @@ func TestRetrieveAttritionTable(t *testing.T) {
requestContext.Params = append(requestContext.Params, gin.Param{Key: "breakdownconceptid", Value: "2"})
requestContext.Writer = new(tests.CustomResponseWriter)
requestContext.Request = new(http.Request)
requestBody := "{\"variables\":[{\"variable_type\": \"concept\", \"concept_id\": 2090006880}," +
"{\"variable_type\": \"custom_dichotomous\", \"provided_name\": \"testABC\", \"cohort_ids\": [1, 3]}," +
requestBody := "{\"variables\":[{\"variable_type\": \"custom_dichotomous\", \"provided_name\": \"testABC\", \"cohort_ids\": [1, 3]}," +
"{\"variable_type\": \"concept\", \"concept_id\": 2090006880}," +
"{\"variable_type\": \"custom_dichotomous\", \"cohort_ids\": [4, 5]}]}" // this one with no provided name (to test auto generated one)
requestContext.Request.Body = io.NopCloser(strings.NewReader(requestBody))
requestContext.Writer = new(tests.CustomResponseWriter)
Expand All @@ -862,9 +851,9 @@ func TestRetrieveAttritionTable(t *testing.T) {
expectedLines := []string{
"Cohort,Size,value1_name,value2_name",
"dummy cohort name,13,5,8",
"Concept C,11,4,7",
"testABC,11,4,7",
"ID_4_5,11,4,7",
"testABC,10,3,7",
"Concept C,9,3,6",
"ID_4_5,8,2,6",
}
i := 0
for _, expectedLine := range expectedLines {
Expand Down
Loading

0 comments on commit 5d0409d

Please sign in to comment.