How to associate cluster labels and dendrogram in the same order on a plot

I'm trying to associate my cluster labels to the corresponding dendrogram, the problem is that the dendrogram does not use the order i want, it's ordering the cluster from the left to the right... I've seen some other post like this one which is a good summary of my problem.

But it's not working as expected for me. I'm using this code :

library("gplots")
library(dendextend)

my_palette <- colorRampPalette(c("grey3", "brown2","burlywood1"))

mytree<-heatmap.2(my_matrix, dendrogram="both", 
                  key=T,hclustfun = function(x) hclust(x,method = 'ward.D2'), col=my_palette, 
                  density.info="histogram", trace="none", cexCol=0.1, cexRow=0.1, main="", 
                  symbreaks=FALSE, scale="none", labRow = rownames(my_matrix))
hc <- as.hclust(mytree$rowDendrogram)

hc.dendo <- as.dendrogram(hc)
my_5_cluster <-cutree(hc, k=5)

    clust.cutree <- dendextend:::cutree(hc.dendo, k=5, order_clusters_as_data = FALSE)
    #idx <- order(as.numeric(names(clust.cutree)))
    #idx <- order(as.numeric((clust.cutree)))
    idx <- order(names(clust.cutree))
    clust.cutree <- clust.cutree[idx]
    tbl <- table(my_5_cluster, clust.cutree)
    lbls <- apply(tbl,2,which.max)
    dend1 <- color_branches(hc.dendo, k = 5, groupLabels = lbls)
    plot(dend1)

and here is the plot I get :

edit:

my data is a 1064x1604 matrix, here is a 50x50 version of it:

structure(c(1, 0.0398830609762799, 0.137176745503742, 0.0221050239579469, 
0.00653808469406079, 0.542027432398559, 0.0398830609762799, 1, 
0.0492335402809855, 0.0238512965576599, 0.226760420903941, 0.00102731032218625, 
0.137176745503742, 0.0492335402809855, 1, 0.354612210304298, 
0.0189498513905832, 0.0527459827865295, 0.0221050239579469, 0.0238512965576599, 
0.354612210304298, 1, 0.0894293105259189, 0.00843951108603055, 
0.00653808469406079, 0.226760420903941, 0.0189498513905832, 0.0894293105259189, 
1, 9.94813097154146e-05, 0.542027432398559, 0.00102731032218625, 
0.0527459827865295, 0.00843951108603055, 9.94813097154146e-05, 
1, 0.292776245084983, 0.154101689187915, 0.973621149798973, 0.424367983931316, 
0.0204733507467241, 0.0839083305915053, 0.0492335402809855, 0.669337992447142, 
0.354612210304296, 0.137176745503742, 0.32268504100422, 0.00599698407571647, 
0.00151860360854231, 0.137176745503742, 0.0127489531549037, 0.137176745503742, 
0.562827025976211, 2.43973117661445e-05, 0.0786770599178212, 
0.0175283743456955, 0.192936362163598, 0.994569461601561, 0.0894293105259189, 
0.00997294185344003, 0.733034465482533, 0.00385593041707034, 
0.137176745503742, 0.0492335402809861, 0.000930172252556588, 
0.442975569236376, 0.669337992447142, 0.0604229474972978, 0.883404322874371, 
0.137176745503742, 0.0371308774639234, 0.238980319150942, 0.669337992447134, 
0.0894293105259189, 0.753788178032326, 0.114644389185804, 0.0149682202529874, 
0.121790361146501, 0.711986988190597, 0.0221050239579469, 0.64788945932501, 
0.0646079496666269, 0.00243949367974157, 0.354612210304298, 0.121790361146503, 
2.75267078421634e-05, 0.00293544932316229, 0.00202209093722833, 
3.49732325112319e-05, 0.182576873423337, 0.406213896654952, 0.0277147384957679, 
0.583864526724944, 0.22676042090394, 0.0117545267432509, 0.129297159910682, 
0.794066571877592, 0.0345462401154051, 0.669337992447142, 0.129297159910682, 
0.0071233971227282, 0.44297556923638, 0.0952516088377655, 0.501313217288423, 
0.371325021381567, 0.0175283743456955, 0.0398830609762796, 0.0117545267432509, 
0.733034465482533, 0.0321207168668463, 0.278579132343827, 0.089429310525918, 
0.0459252410727941, 0.203747109647405, 0.850061011477782, 0.00293544932316229, 
0.251685931289072, 0.0321207168668463, 0.00102731032218624, 0.867187822929886, 
0.154101689187916, 0.182576873423337, 0.7119869881906, 0.292776245084985, 
0.02385129655766, 0.056472414616358, 0.22676042090394, 0.000111373384820816, 
0.0221050239579469, 0.00321692166408852, 3.49732325112319e-05, 
0.462017860138951, 0.00460992400040013, 0.000369549429632312, 
0.0138185624242615, 0.226760420903942, 0.0032169216640885, 0.00167177391165897, 
0.0321207168668463, 0.64788945932501, 0.0786770599178212, 0.481472481715969, 
0.626452348178588, 0.00125064802573173, 0.669337992447142, 0.0839083305915053, 
0.690728577173256, 0.238980319150942, 0.0459252410727941, 0.192936362163598, 
0.371325021381561, 0.0839083305915062, 0.794066571877592, 0.626452348178585, 
0.0690380349665604, 0.129297159910682, 0.0492335402809855, 0.129297159910682, 
0.32268504100422, 0.98476599756469, 0.264883741175431, 0.00599698407571651, 
0.22676042090394, 0.137176745503742, 0.850061011477782, 0.145441002188328, 
0.0298461664571894, 0.0646079496666269, 0.984765997564691, 0.0149682202529871, 
0.338397337158848, 0.101386971222635, 0.00917721284440756, 0.626452348178585, 
0.406213896654952, 0.0786770599178212, 0.542027432398563, 0.481472481715976, 
0.26488374117543, 0.0527459827865301, 0.203747109647403, 0.000139321510537264, 
0.00653808469406075, 0.00653808469406075, 0.000216331276451203, 
0.690728577173256, 0.753788178032326, 0.0238512965576599, 0.711986988190597, 
0.182576873423338, 0.0127489531549039, 0.406213896654952, 0.0321207168668467, 
0.832106590002795, 0.114644389185805, 0.163170386515058, 0.307477821918515, 
0.00653808469406094, 0.0371308774639234, 0.00653808469406085, 
0.0894293105259189, 0.937789756076953, 0.0257188725823969, 0.0221050239579466, 
0.605090978232708, 0.0013785746248032, 0.129297159910682, 0.0604229474972978, 
0.00267687232552811, 0.626452348178588, 0.163170386515058, 0.0398830609762799, 
0.794066571877596, 0.278579132343828, 0.020473350746724, 0.121790361146501, 
0.562827025976211, 0.00137857462480325, 0.154101689187916, 0.0459252410727941, 
0.00151860360854231, 0.883404322874371, 0.832106590002795, 0.00421746990835061, 
0.0839083305915053, 0.0162030370174228, 0.0018392028395664, 0.937789756076951, 
0.406213896654952, 0.424367983931313, 0.521509753085385, 0.129297159910682, 
0.095251608837766, 0.0371308774639232, 0.481472481715976, 0.00125064802573176, 
0.056472414616358, 0.0204733507467241, 0.00151860360854231, 0.462017860138951, 
0.354612210304296, 0.0238512965576599, 0.264883741175431, 0.292776245084984, 
0.0138185624242615, 0.251685931289072, 0.542027432398566, 0.056472414616358, 
0.278579132343827, 0.192936362163598, 0.0127489531549038, 0.354612210304296, 
0.998627326725952, 0.018949851390583, 0.145441002188328, 0.056472414616358, 
0.00653808469406085, 0.690728577173256, 0.690728577173256, 0.00243949367974157, 
0.107847253009764, 0.0459252410727941, 0.000930172252556588, 
0.21501878421584, 0.521509753085385, 0.00460992400040006, 0.137176745503743, 
0.114644389185804, 0.00549710733891162, 0.307477821918513, 0.794066571877599, 
0.00549710733891162, 0.338397337158848, 0.0492335402809861, 0.00167177391165897, 
0.711986988190597, 0.690728577173256, 0.00167177391165897, 0.182576873423337, 
0.0894293105259189, 0.00460992400040009, 0.371325021381567, 0.462017860138947, 
0.0189498513905832, 0.371325021381564, 0.44297556923638, 0.0459252410727938, 
0.203747109647406, 0.958062259992684, 0.0604229474972978, 0.0839083305915053, 
0.0737240501002272, 0.0189498513905832, 0.251685931289077, 0.542027432398563, 
0.0646079496666261, 0.583864526724944, 0.322685041004221, 0.0428115124231191, 
0.145441002188329), .Dim = c(6L, 50L), .Dimnames = list(c("P2018-1129", 
"P2017-0832", "P2017-0997", "P2018-0523", "P2018-1434", "P2018-1304"
), c("X1", "X2", "X3", "X4", "X5", "X6", "X7", "X8", "X9", "X10", 
"X11", "X12", "X13", "X14", "X15", "X16", "X17", "X18", "X19", 
"X20", "X21", "X22", "X23", "X24", "X25", "X26", "X27", "X28", 
"X29", "X30", "X31", "X32", "X33", "X34", "X35", "X36", "X37", 
"X38", "X39", "X40", "X41", "X42", "X43", "X44", "X45", "X46", 
"X47", "X48", "X49", "X50")))

What I expect is to get the number corresponding to each cluster.

I've finally figured out an other way to get my plot like I want, by merging both cutree cluster and denxtend cluster, and then by sorting the df by dendextend order and then getting the order of cutree cluster, by adding this code :

df.merge <- merge(my_5_cluster,clust.cutree,by='row.names')
df.merge.sorted <- df.merge[order(df.merge$y),]
lbls<-unique(df.merge.sorted$x)

Full code is like this :

library("gplots")
library(dendextend)

my_palette <- colorRampPalette(c("grey3", "brown2","burlywood1"))

mytree<-heatmap.2(my_matrix, dendrogram="both", 
                  key=T,hclustfun = function(x) hclust(x,method = 'ward.D2'), col=my_palette, 
                  density.info="histogram", trace="none", cexCol=0.1, cexRow=0.1, main="", 
                  symbreaks=FALSE, scale="none", labRow = rownames(my_matrix))
hc <- as.hclust(mytree$rowDendrogram)

hc.dendo <- as.dendrogram(hc)
my_5_cluster <-cutree(hc, k=5)

    clust.cutree <- dendextend:::cutree(hc.dendo, k=5, order_clusters_as_data = FALSE)

    idx <- order(names(clust.cutree))
    clust.cutree <- clust.cutree[idx]
    df.merge <- merge(my_5_cluster,clust.cutree,by='row.names')
    df.merge.sorted <- df.merge[order(df.merge$y),]
    lbls<-unique(df.merge.sorted$x)
    dend1 <- color_branches(hc.dendo, k = 5, groupLabels = lbls)
    plot(dend1)

and the final figure in the order I wanted: