I have a table with the following data:
| record_id | visit_date | int_date | month | hb_ae | wbc_ae | anc_ae | plt_ae | ast_ae | alt_ae | arc_hb_ae | sched |
|---|---|---|---|---|---|---|---|---|---|---|---|
| 10 | 9/9/2015 | 7/8/2015 | 1 | 2 | Sched | ||||||
| 10 | 10/9/2015 | 7/8/2015 | 2 | 2 | Sched | ||||||
| 10 | 11/9/2015 | 7/8/2015 | 3 | 3 | Sched | ||||||
| 10 | 12/9/2015 | 7/8/2015 | 4 | Unsched | |||||||
| 10 | 1/7/2016 | 7/8/2015 | 5 | 3 | Unsched | ||||||
| 10 | 2/11/2016 | 7/8/2015 | 6 | 2 | Sched | ||||||
| 10 | 3/13/2016 | 7/8/2015 | 7 | Sched | |||||||
| 12 | 11/1/2022 | 10/8/2022 | 2 | 4 | Sched | ||||||
| 12 | 12/3/2022 | 10/8/2022 | 3 | Sched |
I want to do three things:
AEType columns and count those consecutive rows. A null value in that row would count as a stop.My expected result is shown here:
| record_id | AEType | int_date | AEDate | AEs | MaxGrade | sched |
|---|---|---|---|---|---|---|
| 10 | hb_ae | 7/8/2015 | 9/9/2025 | 3 | 3 | Sched |
| 10 | hb_ae | 7/8/2015 | 1/7/2016 | 2 | 3 | Sched |
| 12 | wbc_ae | 10/8/2022 | 11/1/2025 | 1 | 4 | Sched |
I've tried this query, but it's not counting the consecutive values correctly.
Source = table_name,
// Ensure correct types
ChangedTypes = Table.TransformColumnTypes(
Source,
{
{"record_id", type text},
{"visit_date", type date},
{"int_date", type date},
{"sched", type text}
}
),
// Unpivot AE columns
Unpivoted = Table.Unpivot(
ChangedTypes,
{"hb_ae","wbc_ae","anc_ae","plt_ae","ast_ae","alt_ae","arc_hb_ae"},
"AEType",
"Grade"
),
// Remove blanks
Filtered = Table.SelectRows(Unpivoted, each [Grade] <> null),
// Sort by record_id, AEType, visit_date
Sorted = Table.Sort(
Filtered,
{{"record_id", Order.Ascending},
{"AEType", Order.Ascending},
{"visit_date", Order.Ascending}}
),
// Add index per group (record_id + AEType)
WithIndex = Table.Group(
Sorted,
{"record_id", "int_date", "sched", "AEType"},
{
{"AllRows", each Table.AddIndexColumn(_, "RowInGroup", 1, 1, Int64.Type)}
}
),
Expanded = Table.ExpandTableColumn(WithIndex, "AllRows",
{"visit_date","Grade","RowInGroup"}),
// Add DateRank and compute GroupKey = RowInGroup - DateRank
WithKey = Table.Group(
Expanded,
{"record_id","int_date","sched","AEType"},
{
{"WithKey", each
let
t = Table.Sort(_, {{"visit_date", Order.Ascending}}),
t2 = Table.AddIndexColumn(t, "DateRank", 1, 1, Int64.Type),
t3 = Table.AddColumn(t2, "GroupKey", each [RowInGroup] - [DateRank])
in
t3
}
}
),
ExpandedKey = Table.ExpandTableColumn(
WithKey, "WithKey",
{"visit_date","Grade","RowInGroup","DateRank","GroupKey"}
),
// Final grouping: continuous blocks
FinalGrouped = Table.Group(
ExpandedKey,
{"record_id","int_date","sched","AEType","GroupKey"},
{
{"AEDate", each List.Min([visit_date]), type date},
{"AEs", each Table.RowCount(_), Int64.Type},
{"MaxGrade", each List.Max([Grade]), Int64.Type}
}
),
// Clean up
Final = Table.SelectColumns(
FinalGrouped,
{"record_id","int_date","sched","AEType","AEDate","AEs","MaxGrade"}
),
// Sort final output
FinalSorted = Table.Sort(Final,
{{"record_id", Order.Ascending}, {"AEDate", Order.Ascending}})
in
FinalSorted
Any help would be appreciated
Try this, given your new data:
let
Source = Excel.CurrentWorkbook(){[Name="Table1"]}[Content],
#"ae Cols" = List.Select(Table.ColumnNames(Source), each Text.EndsWith(_,"_ae")),
#"Changed Type" = Table.TransformColumnTypes(Source,{
{"record_id", Int64.Type}, {"visit_date", type date}, {"int_date", type date},
{"month", Int64.Type}, {"sched", type text}}),
//don't need month column for the report
#"Removed Columns" = Table.RemoveColumns(#"Changed Type",{"month"}),
//Replace null with <space> so as not to delete null rows when we unpivot
#"Replace Null with Space" = Table.ReplaceValue(#"Removed Columns",null,"",Replacer.ReplaceValue,#"ae Cols"),
#"Unpivoted Other Columns" = Table.UnpivotOtherColumns(#"Replace Null with Space", {"record_id", "visit_date", "int_date", "sched"}, "Attribute", "Value"),
#"Changed Type1" = Table.TransformColumnTypes(#"Unpivoted Other Columns",{{"Value", Int64.Type}}),
//Need to sort in order to use GroupKind.Local in next step
#"Sorted Rows" = Table.Sort(#"Changed Type1",{
{"record_id", Order.Ascending}, {"Attribute", Order.Ascending}, {"int_date", Order.Ascending}, {"visit_date", Order.Ascending}}),
#"Grouped Rows" = Table.Group(#"Sorted Rows", {"record_id", "Attribute","Value"}, {
{"AEType", each Text.Combine(List.RemoveLastN(Text.Split([Attribute]{0},"_"),1),"_"), type text},
//you do not define your logic for `int_date` although they are all the same in your sample
{"int_date", each List.Min([int_date]), type date},
{"AEDate", each List.Min([visit_date]), type date},
{"AEs", each List.Count(List.RemoveNulls([Value])), Int64.Type},
{"MaxGrade", each List.Max([Value]), Int64.Type},
//You do not define your logic to determine "Schedule" vs "Not Scheduled" so next line may need to be edited
{"sched", each List.Last([sched]), type text}
},GroupKind.Local, (x,y)=>Number.From(
x[record_id]<>y[record_id]
or x[Attribute]<>y[Attribute]
or x[Value]=null or y[Value]=null
)
),
#"Removed Columns1" = Table.RemoveColumns(#"Grouped Rows",{"Attribute", "Value"}),
#"Filtered Rows" = Table.SelectRows(#"Removed Columns1", each ([AEs] <> 0))
in
#"Filtered Rows"