Plot table next to plot and below legend in ggplot2

I have this data frame and table:

circ.plot <- structure(list(ID = c("GO:0010257", "GO:0097031", "GO:0032981", 
"GO:0006120", "GO:0046654", "GO:0051123", "GO:0001731", "GO:0000724", 
"GO:0006368", "GO:0006366", "GO:0044068", "GO:0000083", "GO:0034644", 
"GO:0006367", "GO:0016925", "GO:0065004", "GO:0075733", "GO:0030330", 
"GO:0000956", "GO:2001251", "GO:0070646", "GO:0000377", "GO:0000398", 
"GO:0008543", "GO:0006094", "GO:0009267", "GO:2000811", "GO:0046661", 
"GO:0034314", "GO:0071456", "GO:0034976", "GO:0051301", "GO:0030520", 
"GO:0002066", "GO:0001843", "GO:0034599", "GO:0034446", "GO:0030902", 
"GO:0007126", "GO:0051248", "GO:0046488", "GO:0098916", "GO:0099536", 
"GO:0099537", "GO:0007268", "GO:0042592", "GO:0016192", "GO:0009615", 
"GO:0048878", "GO:0007606", "GO:0006812"), term = structure(c(28L, 
25L, 24L, 23L, 46L, 41L, 15L, 13L, 48L, 49L, 26L, 38L, 8L, 50L, 
36L, 37L, 20L, 12L, 33L, 30L, 35L, 42L, 27L, 14L, 16L, 7L, 29L, 
21L, 2L, 5L, 39L, 4L, 19L, 11L, 32L, 6L, 44L, 17L, 22L, 31L, 
34L, 1L, 45L, 47L, 10L, 18L, 51L, 40L, 9L, 43L, 3L), .Label = c("anterograde trans-synaptic signaling", 
"Arp2/3 complex-mediated actin nucleation", "cation transport", 
"cell division", "cellular response to hypoxia", "cellular response to oxidative stress", 
"cellular response to starvation", "cellular response to UV", 
"chemical homeostasis", "chemical synaptic transmission", "columnar/cuboidal epithelial cell development", 
"DNA damage response, signal transduction by p53 class mediator", 
"double-strand break repair via homologous recombination", "fibroblast growth factor receptor signaling pathway", 
"formation of translation preinitiation complex", "gluconeogenesis", 
"hindbrain development", "homeostatic process", "intracellular estrogen receptor signaling pathway", 
"intracellular transport of virus", "male sex differentiation", 
"meiotic nuclear division", "mitochondrial electron transport, NADH to ubiquinone", 
"mitochondrial respiratory chain complex I assembly", "mitochondrial respiratory chain complex I biogenesis", 
"modulation by symbiont of host cellular process", "mRNA splicing, via spliceosome", 
"NADH dehydrogenase complex assembly", "negative regulation of anoikis", 
"negative regulation of chromosome organization", "negative regulation of protein metabolic process", 
"neural tube closure", "nuclear-transcribed mRNA catabolic process", 
"phosphatidylinositol metabolic process", "protein modification by small protein removal", 
"protein sumoylation", "protein-DNA complex assembly", "regulation of transcription involved in G1/S transition of mitotic cell cycle", 
"response to endoplasmic reticulum stress", "response to virus", 
"RNA polymerase II transcriptional preinitiation complex assembly", 
"RNA splicing, via transesterification reactions with bulged adenosine as nucleophile", 
"sensory perception of chemical stimulus", "substrate adhesion-dependent cell spreading", 
"synaptic signaling", "tetrahydrofolate biosynthetic process", 
"trans-synaptic signaling", "transcription elongation from RNA polymerase II promoter", 
"transcription from RNA polymerase II promoter", "transcription initiation from RNA polymerase II promoter", 
"vesicle-mediated transport"), class = "factor"), zscore = c(-3.46410161513775, 
-3.46410161513775, -3.46410161513775, -3.3166247903554, -2, -2, 
-2, -2.23606797749979, -2.82842712474619, -5.75396455568751, 
-2, -2, -2.23606797749979, -3, -2.23606797749979, -3.3166247903554, 
-2, -2.23606797749979, -2.44948974278318, -2.23606797749979, 
-2.82842712474619, -2.12132034355964, -2.12132034355964, -2, 
-2, -2.23606797749979, -1, -2.23606797749979, -2, -2.23606797749979, 
-3, -3.74165738677394, -2, -2, -2, -2.44948974278318, -2.23606797749979, 
-2, -2, -4.12310562561766, -2, -2.44948974278318, -2.44948974278318, 
-2.44948974278318, -2.44948974278318, -4.35889894354067, -4.12310562561766, 
-2, -3.16227766016838, -2, -2.44948974278318), log.pvalue = c(16.4341815462077, 
16.4341815462077, 16.4341815462077, 15.4166395849858, 5.92824818488478, 
2.58701854983776, 2.77229104200152, 1.43213516747591, 4.97971481972944, 
3.01821883926577, 2.50662804866096, 2.50662804866096, 1.91419304117877, 
3.4556013313449, 0.641967092310259, 4.00148992227778, 0.899884391802341, 
0.998141509883199, 0.494769781468246, 0.943462187368159, 2.82923816321647, 
0.674901317134023, 0.674901317134023, 0.402254382343205, 0.798968503025238, 
0.859429723274406, 3.59488384361848, 0.395968939627387, 2.12069030613072, 
0.650772061555318, 1.48001653715317, 1.27340018650023, 1.56027821018946, 
1.24689856450699, 0.569757430240787, 0.313722009606878, 1.48001653715317, 
0.0458121146729004, 0, 0.14904292705079, 0, 0, 0, 0, 0, 0, 0, 
0, 0, 0, 0)), class = "data.frame", row.names = c(NA, -51L))

circ.label <- circ.plot[circ.plot$log.pvalue > 3, ]
circ.table <- circ.label[ ,c("ID","term")]
table1 <- tableGrob(circ.table, rows=NULL, theme=ttheme_minimal())

I have plotted them like this:

p <- ggplot(circ.plot, aes(x = `zscore`, 
                      y = `log.pvalue`,
                      size= `count`)) +
  scale_size(range = c(5, 15)) +
  theme_bw(base_size = 22) + 
  theme(legend.justification= "top",legend.direction="horizontal") +
  geom_point(shape=21,
             alpha=0.5,
             fill="red") +
  guides(fill= FALSE) +
  geom_text_repel(aes(x=`zscore`,y=`log.pvalue`,label=`ID`, size = 3),
                  data=circ.label,
                  nudge_y=c(-2,2),
                  show.legend = FALSE) 

grid.arrange(p,table1,nrow=2, ncol=1, as.table=TRUE)

This is the output:

I want to move the table to the right of the plot and below the legend, for which I have tried this:

grid.arrange(p,table1,nrow=1, ncol=2, as.table=TRUE)

But that gives the following error:

Error in grid.Call(C_convert, x, as.integer(whatfrom), as.integer(whatto), : Viewport has zero dimension(s)`

How can I move the table to the left of the plot and below the legend?

Extract the legend as grob, then use layout matrix, see example:

library(ggplot2)
library(gridExtra)

# using mtcars as an example data
circ.table <- mtcars[1:10, 1:2]
table1 <- tableGrob(circ.table)

# main plot
p <- ggplot(mtcars, aes(x = mpg, y = disp, size= wt)) +
  geom_point() +
  theme(legend.justification = "top", legend.direction = "horizontal")

# extract legend
legend <- cowplot::get_legend(p)

# plot using layout matrix
grid.arrange(p + guides(size = FALSE), # plot without legend
             legend,                   # then add legend as grob
             table1, 
             layout_matrix = rbind(c(1,2),
                                   c(1,3),
                                   c(1,3),
                                   c(1,3),
                                   c(1,3)))

Also, look into patchwork package.