Add a custom element with lines to a ggplot object?

I've made a Kaplan Meier plot using ggplot2 that denotes the disease-free survival (in %) between 3 groups of patients: those unexposed to a risk factor (No exposure), exposed to risk factor 1 (Exposure 1), or those exposed to risk factor 2 (Exposure 2). The data and plot look as follows:

# Load packages
library(tidyverse)
library(magrittr)

# Get data.
data <- 
structure(list(time = c(15.4661190965092, 14.2340862422998, 14.8501026694045, 
9.94934976043806, 4.5119780971937, 7.76454483230664, 12.3121149897331, 
11.7344284736482, 15.6030116358658, 11.9123887748118, 13.9493497604381, 
3.09377138945927, 9.97672826830938, 6.24229979466119, 11.0362765229295, 
12.3805612594114, 9.28405201916495, 3.75633127994524, 6.88021902806297, 
14.2614647501711, 1.95208761122519, 13.3744010951403, 10.4804928131417, 
12.1149897330595, 11.668720054757, 11.211498973306, 5.50855578370979, 
7.61396303901437, 0.939082819986311, 7.0362765229295, 12.911704312115, 
16.0574948665298, 5.34702258726899, 13.3032169746749, 5.637234770705, 
15.0800821355236, 14.1601642710472, 12.7638603696099, 15.6796714579055, 
6.07802874743327, 15.2142368240931, 11.8001368925394, 2.92676249144422, 
10.5051334702259, 10.072553045859, 15.9479808350445, 4.84052019164956, 
4.36960985626283, 12.7775496235455, 15.0280629705681, 13.8891170431211, 
2.2258726899384, 8.90622861054073, 14.7433264887064, 4.61875427789186, 
2.81998631074606, 11.0225872689938, 7.67145790554415, 13.9958932238193, 
9.27310061601643, 15.4496919917864, 11.8986995208761, 11.9972621492129, 
13.7604380561259, 2.8993839835729, 5.82614647501711, 8.0684462696783, 
13.4455852156057, 3.06091718001369, 10.9869952087611, 13.1471594798083, 
14.1273100616016, 13.7577002053388, 6.88021902806297, 11.7973990417522, 
10.3819301848049, 5.47843942505133, 14.2477754962355, 6.94045174537988, 
9.94387405886379, 4.54483230663929, 8.18891170431212, 13.7960301163587, 
8.27104722792608, 14.2422997946612, 14.5325119780972, 14.3655030800821, 
14.072553045859, 3.64681724845996, 6.3791923340178, 10.5434633812457, 
11.378507871321, 14.9322381930185, 14.0123203285421, 2.48323066392882, 
14.8583162217659, 9.88364134154689, 12.145106091718, 15.5838466803559, 
2.80082135523614, 12.9007529089665, 3.50718685831622, 13.3771389459274, 
10.5379876796715, 15.2525667351129, 16.2299794661191, 16.1779603011636, 
8.2217659137577, 15.315537303217, 11.7289527720739, 10.9568788501027, 
13.2676249144422, 7.32375085557837, 11.6112251882272, 5.66735112936345, 
15.7782340862423, 10.113620807666, 9.5523613963039, 1.83983572895277, 
13.5003422313484, 9.6208076659822, 13.9575633127995, 10.9377138945927, 
3.63860369609856, 10.4284736481862, 10.9678302532512, 13.4209445585216, 
15.233401779603, 4.70088980150582, 12.4024640657084, 14.6830937713895, 
13.2977412731006, 0.717316906228611, 12.9691991786448, 7.9315537303217, 
15.6386036960986, 5.82340862422998, 1.0321697467488, 9.89459274469541, 
15.0280629705681, 13.4483230663929, 10.6639288158795, 3.39493497604381, 
12.3203285420945, 4.47364818617385, 14.839151266256, 14.2669404517454, 
6.12183436002738, 13.2101300479124, 3.39493497604381, 15.9671457905544, 
15.9479808350445, 11.9753593429158, 5.46475017111567, 15.9425051334702, 
7.35934291581109, 5.87268993839836, 9.12525667351129, 15.2717316906229, 
6.80082135523614, 14.8583162217659, 12.974674880219, 14.6967830253251, 
13.5797399041752, 11.1321013004791, 3.87679671457906, 14.8336755646817, 
12.5585215605749, 15.6988364134155, 7.60574948665298, 3.40041067761807, 
15.9288158795346, 14.9733059548255, 15.066392881588, 10.2067077344285, 
13.223819301848, 5.34976043805613, 12.0766598220397, 10.9787816563997, 
11.9123887748118, 11.0581793292266, 14.5763175906913, 14.0533880903491, 
4.95003422313484, 4.75290896646133, 14.009582477755, 11.2087611225188, 
14.6776180698152, 15.4086242299795, 10.4421629021218, 6.7378507871321, 
13.9466119096509, 4.22724161533196, 13.7960301163587, 15.7399041752225, 
10.92128678987, 10.425735797399, 11.772758384668, 7.86036960985626, 
2.16837782340862), surv = c(78.895930965634, 82.939817991606, 
81.5385905661846, 90.8500347161838, 96.6504495720828, 91.58932785697, 
84.5274669486008, 87.864369728858, 78.895930965634, 84.8697422025156, 
85.3203796168142, 98.3662199572867, 90.7869444142976, 94.5490389303011, 
89.2078379065114, 84.5274669486008, 90.2944085152522, 98.1562293390666, 
93.3334692912905, 82.939817991606, 98.985749325359, 82.1548721677423, 
90.2111674121547, 84.7006789710365, 87.9377122745415, 86.4721601677476, 
95.7331254544023, 93.6935385308075, 99.6460813737672, 94.5056490644159, 
83.2922253942863, 78.4769648802248, 95.964924789643, 87.1373174726556, 
95.3815962299675, 80.6002593845599, 84.7887574138549, 83.4729026727989, 
80.6002593845599, 95.9831917973004, 80.6002593845599, 89.7929243747343, 
99.0875137586307, 90.2111674121547, 90.7238102387661, 76.5051451787966, 
96.3102631366802, 97.7311888493258, 86.0777214686932, 78.895930965634, 
83.8785010948376, 98.314751291989, 92.5659794066876, 83.781286869923, 
98.2869267301137, 97.7713225402138, 89.2078379065114, 93.6351988430548, 
81.2409555449192, 93.1602670358451, 80.6002593845599, 87.4920568725144, 
87.3419845128403, 81.9494849873229, 99.0875137586307, 95.2639863702512, 
91.0806705136385, 82.1548721677423, 97.661713434227, 89.2078379065114, 
85.655814846093, 83.3214762866605, 84.4919464607611, 94.7936368159517, 
87.7902851000815, 90.2763488914987, 96.7044696855362, 82.939817991606, 
93.3334692912905, 92.5628179141593, 97.5706519561467, 93.1645493591, 
86.0531379063549, 93.1645493591, 84.1840815183361, 82.3344918532314, 
80.6200719805773, 84.7887574138549, 98.1562293390666, 95.363640324494, 
90.1453198447006, 85.9984807940096, 83.781286869923, 81.2409555449192, 
98.6774838813241, 81.5385905661846, 90.8500347161838, 88.9048039353159, 
83.781286869923, 99.0875137586307, 87.9635921093127, 98.1562293390666, 
82.1548721677423, 90.1453198447006, 83.781286869923, 78.4769648802248, 
83.781286869923, 93.1645493591, 80.6002593845599, 89.7929243747343, 
89.2760917993244, 85.2219677930458, 94.1584114615132, 85.2009723473868, 
95.3815962299675, 80.6002593845599, 88.6666313027254, 92.7135716892638, 
99.0367729074855, 82.1548721677423, 91.1603170857887, 81.2409555449192, 
86.9365304945406, 98.1562293390666, 87.9644206553892, 89.2760917993244, 
86.2760734967859, 78.895930965634, 96.3102631366802, 88.346042509788, 
81.9649615708496, 85.1341100942901, 99.7978010383687, 85.9104175149211, 
91.3354416199704, 83.781286869923, 97.331847814204, 99.2663520529959, 
90.8500347161838, 83.781286869923, 82.1548721677423, 87.381862836735, 
98.2615471817051, 87.0324288749706, 98.422121952576, 83.781286869923, 
80.9518418241188, 94.6690250203142, 87.5562613738938, 97.661713434227, 
76.5051451787966, 83.781286869923, 87.3419845128403, 96.7593219088405, 
83.781286869923, 92.3425071953082, 96.3737272908563, 92.2636334459758, 
80.6002593845599, 94.9085034174643, 78.895930965634, 85.8260261421167, 
81.9649615708496, 84.5912319442638, 90.8011947049022, 98.1034003674094, 
78.895930965634, 88.346042509788, 80.6002593845599, 91.9679763619149, 
98.2615471817051, 83.781286869923, 78.895930965634, 80.6002593845599, 
92.5628179141593, 85.3965136911603, 96.9233829481254, 87.2659688605228, 
90.9662877861838, 87.4920568725144, 89.1387381095273, 83.781286869923, 
84.7887574138549, 96.1954714046937, 96.3102631366802, 83.5503943815383, 
88.7910269824191, 82.161520231691, 78.895930965634, 87.9644206553892, 
94.9657807279154, 83.7720562964939, 97.837707856246, 84.3912409226673, 
83.781286869923, 91.129896217454, 90.2763488914987, 85.0368086241741, 
93.4005596740821, 98.314751291989), lower = c(0.755667688497813, 
0.810065890746191, 0.794109376349215, 0.895160242990964, 0.954973873174659, 
0.897551578919665, 0.820019853431481, 0.863148812330468, 0.755667688497813, 
0.823769915532008, 0.82529154050371, 0.978063457368602, 0.894482670798819, 
0.930740060464712, 0.877539994725854, 0.820019853431481, 0.883233189956958, 
0.975612430794298, 0.917004792829363, 0.810065890746191, 0.985444832247924, 
0.794037511553488, 0.888302767494746, 0.821919252439743, 0.863934367913616, 
0.841296374935894, 0.944304104570002, 0.925870604739835, 0.993847072758826, 
0.934730914008468, 0.806501556704641, 0.748465867996515, 0.946984674197919, 
0.845730184148723, 0.940254987103692, 0.782537238695474, 0.819239742335119, 
0.808478164775351, 0.782537238695474, 0.95101679788481, 0.782537238695474, 
0.875434837270998, 0.984169140301171, 0.888302767494746, 0.89380482722086, 
0.710383416866587, 0.950998208465176, 0.970705114470597, 0.844006005507982, 
0.755667688497813, 0.820327282234874, 0.974993450669014, 0.913641913937752, 
0.807389439911771, 0.973679128304006, 0.968331480441595, 0.877539994725854, 
0.925236213381005, 0.783848493168226, 0.913269594557881, 0.782537238695474, 
0.859160372215545, 0.857553709920708, 0.791774046710575, 0.984169140301171, 
0.938905225894942, 0.891921515793743, 0.794037511553488, 0.967006572818361, 
0.877539994725854, 0.839477957536687, 0.814265128963657, 0.82696954977577, 
0.937887498473669, 0.862354931393713, 0.889002473244585, 0.959070284982333, 
0.810065890746191, 0.917004792829363, 0.906527345522126, 0.968867887948537, 
0.920125380198485, 0.83357796263082, 0.920125380198485, 0.812256960507025, 
0.803294477238555, 0.776650436115567, 0.819239742335119, 0.975612430794298, 
0.944158312934169, 0.887595568813524, 0.836106938652997, 0.807389439911771, 
0.783848493168226, 0.981737755020112, 0.794109376349215, 0.895160242990964, 
0.86551624569066, 0.807389439911771, 0.984169140301171, 0.855004479009737, 
0.975612430794298, 0.794037511553488, 0.887595568813524, 0.807389439911771, 
0.748465867996515, 0.807389439911771, 0.920125380198485, 0.782537238695474, 
0.875434837270998, 0.878271586682795, 0.83482024220893, 0.930935218860324, 
0.827392653256551, 0.940254987103692, 0.782537238695474, 0.865333444888939, 
0.908226613283952, 0.98606683967068, 0.794037511553488, 0.898491530076123, 
0.783848493168226, 0.846387163977217, 0.975612430794298, 0.857641356794341, 
0.878271586682795, 0.836077406364583, 0.755667688497813, 0.950998208465176, 
0.859274856520454, 0.799066273544591, 0.833877188985196, 0.996000462948183, 
0.842210678039095, 0.894739107021843, 0.807389439911771, 0.961846153846005, 
0.987263376156533, 0.895160242990964, 0.807389439911771, 0.794037511553488, 
0.851260134210878, 0.976839382161711, 0.854234746739115, 0.975403254711283, 
0.807389439911771, 0.780531389674051, 0.932105479099538, 0.850433122659058, 
0.967006572818361, 0.710383416866587, 0.807389439911771, 0.857553709920708, 
0.9596861872889, 0.807389439911771, 0.905920264839137, 0.95536778457319, 
0.910374592245821, 0.782537238695474, 0.939148873522508, 0.755667688497813, 
0.841304948608203, 0.799066273544591, 0.828043753101048, 0.886705435390677, 
0.974998731884024, 0.755667688497813, 0.859274856520454, 0.782537238695474, 
0.901754541297081, 0.976839382161711, 0.807389439911771, 0.755667688497813, 
0.782537238695474, 0.906527345522126, 0.836693897077852, 0.961531714133711, 
0.856739455820482, 0.888556119004382, 0.859160372215545, 0.876798844500371, 
0.807389439911771, 0.819239742335119, 0.94966129601496, 0.950998208465176, 
0.816760657267524, 0.873071386340173, 0.801332622518322, 0.755667688497813, 
0.857641356794341, 0.939778275538934, 0.819172581880608, 0.971928452490634, 
0.825879122468099, 0.807389439911771, 0.890390776160883, 0.889002473244585, 
0.825598067911455, 0.922686519490444, 0.974993450669014), upper = c(0.823717623193319, 
0.849191835758465, 0.837232495866684, 0.922039251916947, 0.978174342239545, 
0.934609795616254, 0.871307278579666, 0.894416739913637, 0.823717623193319, 
0.874379241789826, 0.882060074614875, 0.989292990734727, 0.921456563124246, 
0.960474480724528, 0.906857623787338, 0.871307278579666, 0.923094863488607, 
0.987548441774093, 0.949955393697625, 0.849191835758465, 0.994289913434519, 
0.850013119366769, 0.916135243934375, 0.872860076809132, 0.895095915532934, 
0.88879908517927, 0.970538120602751, 0.948132397495287, 0.999081428653254, 
0.955496129553027, 0.860208483599158, 0.822834317521706, 0.972483192263057, 
0.89779367446523, 0.967572522789849, 0.830171586937913, 0.87753718628127, 
0.8618322403993, 0.830171586937913, 0.968728746756924, 0.830171586937913, 
0.921001646781718, 0.997626828642714, 0.916135243934375, 0.920873270491501, 
0.823926502204367, 0.975361120861275, 0.983963629274964, 0.877881683884755, 
0.823717623193319, 0.857658046767516, 0.991369769199295, 0.937835755212857, 
0.869382689764604, 0.992146148072156, 0.987185866032483, 0.906857623787338, 
0.947601308247541, 0.842011296235924, 0.950303766369387, 0.830171586937913, 
0.890969865852105, 0.88957952958382, 0.848186186145897, 0.997626828642714, 
0.966575416650932, 0.930091761922735, 0.850013119366769, 0.986323210100933, 
0.906857623787338, 0.873985856457408, 0.852605393948829, 0.863258994078296, 
0.958092905110497, 0.893731093471867, 0.916737513612901, 0.975085413821684, 
0.849191835758465, 0.949955393697625, 0.94513147369813, 0.982593420791875, 
0.943309840601488, 0.888356323643571, 0.943309840601488, 0.8725021668835, 
0.843895824110921, 0.836875343643832, 0.87753718628127, 0.987548441774093, 
0.963209640942235, 0.91552718100721, 0.884544590766402, 0.869382689764604, 
0.842011296235924, 0.991837766792364, 0.837232495866684, 0.922039251916947, 
0.913219619173031, 0.869382689764604, 0.997626828642714, 0.904976959388002, 
0.987548441774093, 0.850013119366769, 0.91552718100721, 0.869382689764604, 
0.822834317521706, 0.869382689764604, 0.943309840601488, 0.830171586937913, 
0.921001646781718, 0.907489287802726, 0.869981755030479, 0.952354822262432, 
0.877359215164471, 0.967572522789849, 0.830171586937913, 0.908525095500321, 
0.946438504405819, 0.994687377501161, 0.850013119366769, 0.924906149140602, 
0.842011296235924, 0.892967267947796, 0.987548441774093, 0.902211540982596, 
0.907489287802726, 0.890295659392204, 0.823717623193319, 0.975361120861275, 
0.908326732466828, 0.840763169181104, 0.869170760069249, 0.999959484216824, 
0.876336530756437, 0.932357022337168, 0.869382689764604, 0.984927637444616, 
0.998093202674114, 0.922039251916947, 0.869382689764604, 0.850013119366769, 
0.896974925284878, 0.988425715717517, 0.886716877859586, 0.993118901629567, 
0.869382689764604, 0.839581954218875, 0.961503231045787, 0.901434657413618, 
0.986323210100933, 0.823926502204367, 0.869382689764604, 0.88957952958382, 
0.975565398383733, 0.869382689764604, 0.941268118848152, 0.972180082048895, 
0.935063228824689, 0.830171586937913, 0.959126319042212, 0.823717623193319, 
0.875557284612824, 0.840763169181104, 0.86416647611313, 0.929830429674193, 
0.987106633980016, 0.823717623193319, 0.908326732466828, 0.830171586937913, 
0.937961306403804, 0.988425715717517, 0.869382689764604, 0.823717623193319, 
0.830171586937913, 0.94513147369813, 0.87159289389749, 0.976997640745795, 
0.888875756734311, 0.93127100659334, 0.890969865852105, 0.906218647708942, 
0.869382689764604, 0.87753718628127, 0.974407271055667, 0.975361120861275, 
0.854677357338004, 0.903001357728714, 0.842411155753018, 0.823717623193319, 
0.902211540982596, 0.959641198780709, 0.856688513674614, 0.984868490477327, 
0.862339457520645, 0.869382689764604, 0.932698114912117, 0.916737513612901, 
0.875881267415946, 0.945463531021287, 0.991369769199295), group = c("Exposure 3", 
"Exposure 1", "Exposure 1", "Exposure 1", "Exposure 3", "Exposure 3", 
"Exposure 3", "Exposure 1", "Exposure 3", "Exposure 3", "Exposure 2", 
"Exposure 1", "Exposure 1", "Exposure 3", "Exposure 1", "Exposure 3", 
"Exposure 3", "Exposure 1", "Exposure 3", "Exposure 1", "Exposure 1", 
"Exposure 3", "Exposure 1", "Exposure 3", "Exposure 1", "Exposure 3", 
"Exposure 3", "Exposure 1", "Exposure 1", "Exposure 1", "Exposure 3", 
"Exposure 1", "Exposure 3", "Exposure 2", "Exposure 3", "Exposure 1", 
"Exposure 2", "Exposure 3", "Exposure 1", "Exposure 1", "Exposure 1", 
"Exposure 2", "Exposure 2", "Exposure 1", "Exposure 1", "Exposure 3", 
"Exposure 3", "Exposure 1", "Exposure 1", "Exposure 3", "Exposure 1", 
"Exposure 3", "Exposure 1", "Exposure 2", "Exposure 2", "Exposure 3", 
"Exposure 1", "Exposure 1", "Exposure 3", "Exposure 2", "Exposure 1", 
"Exposure 1", "Exposure 1", "Exposure 3", "Exposure 2", "Exposure 3", 
"Exposure 3", "Exposure 3", "Exposure 3", "Exposure 1", "Exposure 1", 
"Exposure 1", "Exposure 1", "Exposure 1", "Exposure 1", "Exposure 1", 
"Exposure 1", "Exposure 1", "Exposure 3", "Exposure 2", "Exposure 1", 
"Exposure 1", "Exposure 2", "Exposure 1", "Exposure 2", "Exposure 1", 
"Exposure 3", "Exposure 2", "Exposure 1", "Exposure 1", "Exposure 1", 
"Exposure 3", "Exposure 2", "Exposure 3", "Exposure 1", "Exposure 1", 
"Exposure 1", "Exposure 2", "Exposure 2", "Exposure 2", "Exposure 2", 
"Exposure 1", "Exposure 3", "Exposure 1", "Exposure 2", "Exposure 1", 
"Exposure 2", "Exposure 1", "Exposure 1", "Exposure 2", "Exposure 1", 
"Exposure 1", "Exposure 1", "Exposure 3", "Exposure 3", "Exposure 1", 
"Exposure 3", "Exposure 2", "Exposure 1", "Exposure 3", "Exposure 1", 
"Exposure 3", "Exposure 3", "Exposure 1", "Exposure 3", "Exposure 1", 
"Exposure 2", "Exposure 3", "Exposure 3", "Exposure 2", "Exposure 1", 
"Exposure 1", "Exposure 1", "Exposure 1", "Exposure 3", "Exposure 2", 
"Exposure 2", "Exposure 3", "Exposure 1", "Exposure 2", "Exposure 3", 
"Exposure 3", "Exposure 1", "Exposure 1", "Exposure 2", "Exposure 2", 
"Exposure 3", "Exposure 3", "Exposure 2", "Exposure 3", "Exposure 3", 
"Exposure 2", "Exposure 1", "Exposure 1", "Exposure 2", "Exposure 3", 
"Exposure 1", "Exposure 1", "Exposure 1", "Exposure 1", "Exposure 3", 
"Exposure 1", "Exposure 1", "Exposure 1", "Exposure 2", "Exposure 1", 
"Exposure 3", "Exposure 2", "Exposure 1", "Exposure 3", "Exposure 1", 
"Exposure 2", "Exposure 3", "Exposure 1", "Exposure 2", "Exposure 1", 
"Exposure 1", "Exposure 1", "Exposure 2", "Exposure 1", "Exposure 1", 
"Exposure 2", "Exposure 2", "Exposure 3", "Exposure 3", "Exposure 1", 
"Exposure 1", "Exposure 1", "Exposure 3", "Exposure 3", "Exposure 1", 
"Exposure 1", "Exposure 1", "Exposure 1", "Exposure 2", "Exposure 2", 
"Exposure 1", "Exposure 3", "Exposure 1", "Exposure 3")), class = "data.frame", row.names = c(NA, 
-200L))

# Make the plot:
data %>% 
  ggplot(aes(x=time, y=surv)) + 
  geom_line(aes(color=group), size=0.3) + 
  scale_y_continuous(name="Disease-free survival (%)") +
  scale_x_continuous(name="Follow-up time (years)") +
  scale_color_manual(name="Groups", 
                     values=c("Exposure 1"="firebrick3", "Exposure 2"="#255B1D", "Exposure 3"="darkblue"), 
                     labels=c("Exposure 1 (p<.01)", "Exposure 2 (p=.02)", "Exposure 3 (p=.03)")) + 
  theme_classic() + 
  theme(legend.position=c(0.3, 0.3))

This produces the following plot:

In this figure the p-values denote the difference in survival between each group and the unexposed group (using Log Rank tests). Now I've also compared the survival between the two exposed groups, Exposure 1 and Exposure 2, for which the p-value was p=.01. I'd like to add this to the plot in the following manner:

I've made this (very ugly) adaptation in Paint, but I'd like to be able to do this in R. Any help?

Perhaps the easiest thing would be to draw your own legend, so that you don't need to guess where to put the lines, and so that relative positions stay constant during plot resizing:

data %>% 
  ggplot(aes(x=time, y=surv)) + 
  geom_line(aes(color=group), size=0.3) + 
  scale_y_continuous(name = "Disease-free survival (%)") +
  scale_x_continuous(name = "Follow-up time (years)") +
  scale_color_manual(name = "Groups", 
                     values=c("firebrick3", "#255B1D", "darkblue")) +
  geom_line(data = data.frame(time = c(3.9, 5, 3.9, 5, 3.9, 5),
                              surv = c(84, 84, 83, 83, 82, 82),
                              group = rep(paste('Exposure', 1:3), each = 2)),
            aes(color = group)) +
  geom_text(data = data.frame(time = c(5.3, 5.3, 5.3, 10), 
                              surv = c(84, 83, 82, 82.5),
                              group = c("No Exposure", "Exposure 1 (p = 0.02)",
                                        "Exposure 2 (p = 0.03)", "p = 0.01")),
            aes(label = group), hjust = 0, size = 3.2) +
  geom_path(data = data.frame(time = c(9, 9.5, 9.5, 9), 
                              surv = c(82, 82, 83, 83))) +
  annotate('text', x = 3.8, y = 85.2, label = "Groups", hjust = 0, size = 4) +
  theme_classic() + 
  theme(legend.position = 'none')