sqlduckdb

DuckDB last function does not return last value


I can't understand why my SQL doesn't work

INSERT INTO per_sec_without_gaps
    SELECT
        row_number() OVER() AS Index,
        UTCDateTime AS TimeKey,
        (SELECT CASE WHEN CAST(without_gaps.UTCDateTime AS TIME) > CAST(min(UTCDateTime) AS TIME) 
                    THEN first(UTCTradeDate) ELSE last(UTCTradeDate) END FROM per_sec),
        CAST(UTCDateTime AS TIME),
        (SELECT CASE WHEN CAST(without_gaps.LocalDateTime AS TIME) > CAST(min(LocalDateTime) AS TIME) 
                    THEN first(LocalTradeDate) ELSE last(LocalTradeDate) END FROM per_sec),
        CAST(LocalDateTime AS TIME),
        (SELECT last(Ticker) FILTER(Ticker IS NOT NULL AND UTCDateTime <= TimeKey) FROM per_sec),
        (SELECT last(BidPrice) FILTER(BidPrice IS NOT NULL AND UTCDateTime <= TimeKey) FROM per_sec),
        (SELECT last(AskPrice) FILTER(AskPrice IS NOT NULL AND UTCDateTime <= TimeKey) FROM per_sec),
        NULL,
        (SELECT last(BidQuantity) FILTER(BidQuantity IS NOT NULL AND UTCDateTime <= TimeKey) FROM per_sec),
        (SELECT last(AskQuantity) FILTER(AskQuantity IS NOT NULL AND UTCDateTime <= TimeKey) FROM per_sec),
        NULL
    FROM without_gaps

The most strange thing is that sometimes it works correct but sometime not, I mean with the same input it can produce different output and I have no idea what depends on it

I try different approaches - Window function JOIN before do it but nothing works


Solution

  • I've found where the problem was. Kliment Merzlyakov was right regarding ordering data, but the right solution is a little bit different, I didn't post that part where the problem was, but thanks to Kliments' answer I paid attention to the ordering

    The part where the problem was is:

    INSERT INTO per_sec
    SELECT
        last(UTCDate),
        UTCTime AS UTCTimeBarStartSec,
        last(LocalDate),
        LocalTime AS LocalTimeBarStart,
        last(Ticker) AS Ticker,
        last(BidPrice) FILTER(BidPrice IS NOT NULL),
        last(AskPrice) FILTER(AskPrice IS NOT NULL),
        last(BidQuantity) FILTER(BidQuantity IS NOT NULL),
        last(AskQuantity) FILTER(AskQuantity IS NOT NULL),
        last(UTCDateTime) AS UTCDateTime,
        last(LocalDateTime)
    FROM raw_data GROUP BY UTCTimeBarStartSec, LocalTimeBarStart
    ORDER BY UTCDateTime;
    

    The right solution is:

    INSERT INTO per_sec
    SELECT
        last(UTCDate ORDER BY Index),
        UTCTime AS UTCTimeBarStartSec,
        last(LocalDate ORDER BY Index),
        last(LocalTime ORDER BY Index) AS LocalTimeBarStart,
        last(Ticker ORDER BY Index) AS Ticker,
        last(BidPrice ORDER BY Index) FILTER(BidPrice IS NOT NULL),
        last(AskPrice ORDER BY Index) FILTER(AskPrice IS NOT NULL),
        last(BidQuantity ORDER BY Index) FILTER(BidQuantity IS NOT NULL),
        last(AskQuantity ORDER BY Index) FILTER(AskQuantity IS NOT NULL),
        last(UTCDateTime ORDER BY Index) AS UTCDateTime,
        last(LocalDateTime ORDER BY Index)
    FROM raw_data GROUP BY UTCTimeBarStartSec
    ORDER BY UTCDateTime;
    

    Problem was regarding grouping and ordering. But grouping all data and ordering only in last SQL function