I am trying to fine tune a T5 model for more accurate summarization, but my loss is very high and does not change with each epoch. I have tried increasing the learning rate, but the model still does not train. It seems like there is some issue with the code since the loss doesn't change at all. My input texts are very large, but I thought this would be fine given that T5 can already be used for summarization.
training code:
import torch
from torchtext.models import T5_BASE_GENERATION, T5Transform
from torchtext.prototype.generate import GenerationUtils
from torch.utils.data import DataLoader, Dataset
import torch.nn.functional as F
import json
import os
padding_idx = 0
eos_idx = 0
max_seq_len = 65536 #16384
t5_sp_model_path = "/t5_tokenizer_base.model"
# Define your custom dataset class
class CustomDataset(Dataset):
def __init__(self, data):
self.data = data
def __getitem__(self, index):
example = self.data[index]
input_text = example["input_text"]
target_text = example["target_text"]
return input_text, target_text
def __len__(self):
return len(self.data)
# Load labeled dataset
train_data = [] # List of labeled training examples
valid_data = [] # List of labeled validation examples
labeled = json.loads(open("data.json", 'r').read())
datalen = len(labeled)
valper = 0.2
train_ind = range(datalen - int(datalen * valper))
valid_ind = range(datalen - int(datalen * valper), datalen)
for x in train_ind:
train_data.append(labeled[str(x)])
for x in valid_ind:
valid_data.append(labeled[str(x)])
# Create instances of the T5 model and transformation
if os.path.exists('model.pt'):
t5_base = torch.load("model.pt")
else:
t5_base = T5_BASE_GENERATION.get_model()
transform = T5Transform(sp_model_path=t5_sp_model_path, max_seq_len=max_seq_len, eos_idx=eos_idx, padding_idx=padding_idx)
# Define the training parameters
device = torch.device("cuda") # if torch.cuda.is_available() else "cpu")
batch_size = 1
num_epochs = 5
# Convert your training and validation data into tensors
train_dataset = CustomDataset(train_data)
valid_dataset = CustomDataset(valid_data)
train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
valid_dataloader = DataLoader(valid_dataset, batch_size=batch_size)
# Define the loss function and optimizer
criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(t5_base.parameters(), lr=0.001)
# Training loop
t5_base.train()
t5_base.to(device)
for epoch in range(num_epochs):
total_loss = 0
total_batches = 0
for batch in train_dataloader:
input_batch = []
target_batch = []
for input_text, target_text in zip(batch[0], batch[1]):
input_batch.append('summarize: ' + input_text[:5000])
target_batch.append(target_text)
#print(input_batch)
#print(target_batch)
#print('\n\n\n')
input_batch = transform(input_batch)
target_batch = transform(target_batch)
input_batch = input_batch.to(device)
target_batch = target_batch.to(device)
optimizer.zero_grad()
sequence_generator = GenerationUtils(t5_base)
sequence_generator.device = device
beam_size = 1
output = sequence_generator.generate(input_batch, eos_idx=eos_idx, num_beams=beam_size, max_length=30)
#print(' ')
#print(output)
#print('OUTPUT:', transform.decode(output.tolist()))
#print('TARGET:', target_batch)
#print('\n')
logits = output.float().squeeze()
log_size = int(logits.numel())
#print(logits)
#print(logits.shape)
#print('\n\n')
target = target_batch.view(-1)
tar_size = int(target.numel())
#print(target)
#print(target.shape)
if log_size > tar_size:
#print('Target Shorter')
target = torch.nn.functional.pad(target, (1, log_size - tar_size - 1)).float()
else:
#print('Input Shorter')
logits = torch.nn.functional.pad(logits, (0, tar_size - log_size)).float()
#print('\n')
#print(logits)
#print(logits.shape)
#print(target)
#print(target.shape)
#print('\n\n')
loss = criterion(logits, target)
#print('LOSS:', loss)
loss.requires_grad = True
loss.backward()
optimizer.step()
#print(f"Completed Batch #{total_batches}")
#print('\n')
total_loss += loss.item()
total_batches += 1
#print(total_loss, total_batches)
average_loss = total_loss / total_batches
print(f"Epoch {epoch + 1}/{num_epochs}, Loss: {average_loss}")
print('------------------------------------------------------')
print('\n')
torch.save(t5_base, "model.pt")
result:
Epoch 1/5, Loss: 1139705410.5365853
------------------------------------------------------
Epoch 2/5, Loss: 1139705410.5365853
------------------------------------------------------
Epoch 3/5, Loss: 1139705410.5365853
------------------------------------------------------
Epoch 4/5, Loss: 1139705410.5365853
------------------------------------------------------
Epoch 5/5, Loss: 1139705410.5365853
------------------------------------------------------
sample data.json (news article summary):
{
"0": {
"input_text": "Ukrainian President Volodymyr Zelenskiy attended a summit of the Arab League in Saudi Arabia on Friday to canvas support for his people, while Saudi Crown Prince Mohammed bin Salman expressed his readiness to mediate in the war between Moscow and Kyiv. Also at the Jeddah gathering, Arab leaders warmly welcomed back into their fold Syria\u2019s President Bashar al-Assad \u2014 who has received heavy support from Russia in his country\u2019s civil war \u2014 following a decade of isolation. \u201cWe reaffirm the kingdom\u2019s readiness to continue mediating efforts between Russia and Ukraine, and to support all international efforts aimed at resolving the crisis politically in a way that contributes to achieving security,\u201d the Saudi Crown Prince said in his opening speech. Prince Mohammed has mediated in the conflict before. Zelenskiy, who was also due to attend a summit of the G7 leaders in the Japanese city of Hiroshima this weekend, thanked Saudi Arabia for its past help and said delegates would each receive the text of his 10-point peace plan. He asked them to work with Ukraine directly without intermediaries. Gulf states have tried to remain neutral in the Ukraine conflict despite Western pressure on Gulf oil producers to help isolate Russia, a fellow OPEC+ member. Saving people In his address to the summit, Zelenskiy said some countries including members of the Arab League preferred to \u201cturn a blind eye\u201d to Russia\u2019s illegal annexation of Ukrainian land and to its jailing of some Ukrainians during the 15-month war. \u201cI am sure we can all be united in saving people from the cages of Russian prisons,\u201d he said, speaking in English. Last year, in a diplomatic coup, Crown Prince Mohammed secured the release of 10 foreigners captured by Russia in Ukraine. The move was apparently made possible by his close ties with Russian President Vladimir Putin. \u201cThe Kingdom of Saudi Arabia plays a significant role and we are ready to take our cooperation to a new level,\u201d Zelenskiy said wrote on Twitter shortly after arriving in Jeddah. Saudi Arabia faced heavy criticism from the United States over an OPEC+ decision to cut oil production, seen as helping Russia to refill its coffers by boosting prices. Even though the October decision initially drew the ire of the United States and other Western countries, market dynamics since then have shown the cuts to be prudent. At a time when Russia\u2019s war on Ukraine has roiled global energy markets, the role the kingdom plays as the world\u2019s largest oil exporter has grown in importance to both Washington and Moscow. KYIV, Ukraine (AP) \u2014 Ukrainian President Volodymyr Zelenskyy addressed a summit of Arab leaders in Saudi Arabia on Friday before what a senior official said would be a trip to Japan for a meeting with the leaders of the world\u2019s most powerful democracies. Zelenskyy has in recent months made foreign trips to shore up diplomatic support for Ukraine\u2019s fight against Russia\u2019s full-scale invasion almost 15 months ago and solicit more military support. He earlier this week returned from a three-day trip to Italy, the Vatican, Germany, France and the United Kingdom. Ukraine and Russia are squaring up for a major and potentially decisive phase of the war as Kyiv prepares an expected counteroffensive. The conflict has been bogged down in a war of attrition in recent months amid bad weather. Zelenskyy\u2019s office said he was invited to attend the Arab League summit in Jeddah, where he met with Saudi Crown Prince Mohammed bin Salman before holding other bilateral meetings. They discussed Zelenskyy\u2019s peace plan, the security situation in Ukraine and possible investments in the reconstruction of the country, a presidential statement said. Zelenskyy also invited Prince Mohammed to visit Ukraine. Zelenskyy urged leaders at the summit to resist Moscow\u2019s influence and consider his peace proposals, which include the withdrawal of the Kremlin\u2019s forces from occupied areas of Ukraine. \u201cI\u2019m more than sure that none of you will agree to surrender a third of your country to the invaders,\u201d Zelenskyy said in English. \u201cAnother priority is the protection of the Muslim community of Ukraine,\u201d Zelenskyy said. \u201cCrimea was the first to suffer from the Russian occupation, and most of those who suffer repression in occupied Crimea are Muslims.\u201d Crimean Tatar leader Mustafa Dzhemilev accompanied Zelenskyy on the visit. Zelenskyy will later travel to a Group of Seven summit in Japan, where leaders of the world\u2019s most powerful democracies aim to step up punishment on Russia for its full-scale invasion of Ukraine, according to Oleksiy Danilov, the secretary of Ukraine\u2019s National Security and Defense Council. However, Danilov\u2019s office later posted a statement backtracking on his announcement and saying Zelenskyy would appear at the G-7 summit via video. Zelenskyy\u2019s movements are kept secret for security reasons. Meanwhile, Russian forces kept up their long-range bombardment of Ukrainian targets while drones reportedly damaged train lines behind their front line. About 130 meters (430 feet) of railway track were damaged and trains were halted for hours after an explosion derailed eight cars of a freight train carrying grain in Russia-occupied Crimea, Russian state media reported Friday. Thursday\u2019s blast prompted renewed suspicions about possible Ukrainian saboteur activity behind Russian lines. Train traffic was also halted in northern Crimea on Thursday night after a drone hit a railway track near the town of Dzhankoi, Russia\u2019s Baza Telegram channel reported. Sergei Aksyonov, the Kremlin-appointed head of Crimea, said in a separate post that four Ukrainian drones were shot down overnight in the peninsula\u2019s north. Aksyonov claimed there was no damage or casualties. Russia overnight fired cruise missiles, drones and artillery at targets across Ukraine, killing two civilians, officials said Friday. The attacks included an air assault on Kyiv for the second straight day and the 10th time in three weeks. The Kremlin\u2019s forces also took aim at central, eastern and southern Ukraine, and the western Lviv region near the border with Poland. Russia launched 22 Iranian-made Shahed drones and six Kalibr cruise missiles during the night, the Ukrainian Air Force said. It said air defenses downed 16 drones and three missiles. The Russian shelling killed two civilians and wounded nine others in Ukraine\u2019s eastern Donetsk region, said its governor, Pavlo Kyrylenko. The missile attacks that have intensified recently aim to \u201cdisrupt Ukraine\u2019s plans and preparations for active military operations during the spring-summer campaign,\u201d according to a statement from Ukraine\u2019s intelligence agency, published on Telegram. The targets are Ukraine\u2019s military control points and barracks, supply routes and the places where ammunition, equipment, fuel are stored, it said. On Friday, the United Nations said operations to ship Ukrainian grain were \u201cpartially restarting,\u201d two days after Russia gave a green light to extend the deal for two months. The U.N. also urged a swift return to the previous tempo of ship arrivals and departures from all three Black Sea ports and inspections of their cargo. U.N. associate spokesperson Stephanie Tremblay said the Joint Coordination Center, which includes representatives from the four parties involved in the deal \u2013 Russia, Ukraine, Turkey and the United Nations -- approved the registration Friday of six new vessels to participate in the grain shipments. Nine applications to participate remain pending, she said. No ships are currently loading at any of the three ports, Tremblay said, but inspection teams from the center checked and cleared three new vessels Friday to proceed to the ports of Odesa and Chornomorsk. ___ Hanna Arhirova in Kyiv and Edith M. Lederer at the United Nations contributed to this report. ___ Follow AP\u2019s coverage of the war in Ukraine at https://apnews.com/hub/russia-ukraine President Zelenskyy makes a stop in Saudi Arabia on his way to Japan to meet with the G7. At the G7 increased sanctions against Russia are on the table, but negotiations still continue. A look at the battle in Bakhmut plus Crimean Tartars; expelled by the Soviets decades ago, now looking to return. TALLINN, Estonia (AP) \u2014 While the world awaits Ukraine\u2019s spring battlefield offensive, its leader, Volodymyr Zelenskyy, has launched a diplomatic one. In the span of a week, he's dashed to Italy, the Vatican, Germany, France and Britain to shore up support for defending his country. On Friday, he was in Saudi Arabia to meet with Arab leaders, some of whom are allies with Moscow. President Vladimir Putin, meanwhile, was in the southern Russian city of Pyatigorsk, chairing a meeting with local officials, sitting at a large table at a distance from the other attendees. The Russian president has faced unprecedented international isolation, with an International Criminal Court arrest warrant hanging over his head and clouding the prospects of traveling to many destinations, including those viewed as Moscow's allies. With his invasion of Ukraine, \u201cPutin took a gamble and lost really, really big time,\u201d said Theresa Fallon, director of the Brussels-based Centre for Russia Europe Asia Studies. \u201cHe is an international pariah, really.\u201d It was only 10 years ago when Putin stood proudly among his peers at the time -\u2013 Barack Obama, Angela Merkel and Shinzo Abe \u2013 at a Group of Eight summit in Northern Ireland. Russia has since been kicked out of the group, which consists of Canada, France, Germany, Italy, Japan, Britain and the United States, for illegally annexing Crimea in 2014. Now it appears to be Ukraine\u2019s turn in the spotlight. There were conflicting messages from Kyiv whether Zelenskyy would attend the G7 in Japan on Sunday. The secretary of Ukraine\u2019s National Security and Defense Council said on national television the president would be there, but the council later walked back those remarks, saying Zelenskyy would join via video link. The president\u2019s office would not confirm either way for security reasons. But whether in person or via video, it would be of great symbolic and geopolitical significance. \u201cIt conveys the fact that the G7 continues to strongly support Ukraine,\u201d said Nigel Gould-Davies, senior fellow for Russia and Eurasia at the International Institute for Strategic Studies. \u201cIt\u2019s a visible marker of the continued commitment of the most highly industrialized and highly developed countries in the world.\u201d Story continues It also comes at a time when the optics are just not in the Kremlin\u2019s favor. There\u2019s uncertainty over whether Putin can travel to South Africa in August for a summit of the BRICS nations of Brazil, Russia, India, China and South Africa. Moscow has long showcased the alliance as an alternative to the West\u2019s global dominance, but this year it is already proving awkward for the Kremlin. South Africa, the host of the summit, is a signatory to the ICC and is obligated to comply with the arrest warrant on war crimes charges. South Africa has not announced that Putin will definitely come to the summit but has been planning for his possible arrival. South African President Cyril Ramaphosa has appointed an inter-ministerial committee, led by Deputy President Paul Mashatile, to consider South Africa\u2019s options with regard to its ICC commitment over Putin\u2019s possible trip. While it is highly unlikely the Russian president would be arrested there if he decides to go, the public debate about whether he can is in itself \u201can unwelcome development whose impact should not be underestimated,\u201d according to Gould-Davies. Then there are Moscow\u2019s complicated relations with its own neighbors. Ten days ago, Putin projected the image of solidarity, with leaders of Armenia, Belarus and Central Asian states standing beside him at a Victory Day military parade on Red Square. This week, however, the leaders of Kazakhstan, Kyrgyzstan, Tajikistan, Turkmenistan and Uzbekistan flocked to China and met with leader Xi Jinping at a summit that highlighted the erosion of Russia\u2019s influence in the region as Beijing seeks to make economic inroads into Central Asia. Xi is using the opportunity \u201cof a weakened Russia, a distracted Russia, almost a pariah-state Russia to increase (China\u2019s) influence in the region,\u201d Fallon said. Putin\u2019s effort this month to shore up more friends in the South Caucasus by scrapping visa requirements for Georgian nationals and lifting a four-year ban on direct flights to the country also didn\u2019t appear to go as smoothly as the Kremlin may have hoped. The first flight that landed Friday in Georgia was met with protests, and the country\u2019s pro-Western president has decried the move as a provocation. Zelenskyy\u2019s ongoing world tour can be seen as a success on many levels. Invitations from other world leaders is a sign they think Ukraine is \"going to come out of the war in good shape,\u201d said Phillips P. O\u2019Brien, professor of strategic studies at the University of St. Andrews in Scotland. Otherwise, \u201cit simply wouldn\u2019t be happening,\u201d he said. \"No one would want to be around a leader they think is going to be defeated and a country that\u2019s going to collapse.\u201d By contrast, the ICC warrant might make it harder for leaders even to visit Putin in Moscow because \u201cit\u2019s not a good look to visit an indicted war criminal,\u201d Gould-Davies said. European leaders promised him an arsenal of missiles, tanks and drones, and even though no commitment has been made on fighter jets \u2013 something Kyiv has wanted for months \u2013 a conversation about finding ways to do it has begun. His appearance Friday at the Arab League summit in Jeddah, a Saudi Arabian port on the Red Sea, highlighted Kyiv\u2019s effort to spread its plight for support far and wide, including in some countries whose sympathies are with Russia. In addition to Zelenskyy, Saudi Crown Prince Mohammed bin Salman also welcomed Syrian President Bashar Assad at the summit after a 12-year suspension \u2013 something analysts say aligns with Moscow\u2019s interests. Anna Borshchevskaya, a senior fellow at the Washington Institute who focuses on Russia\u2019s policy in the Middle East, called it \u201canother testament to the fact that Russia is not isolated globally for its invasion of Ukraine, that the Middle East is one part of the world where Russia is able to find avenues to avoid global isolation \u2013 both ideological isolation but also economic isolation.\u201d She added that Zelenskyy and his government deserve credit for \u201cin recognizing that they need to reach out more to improve their diplomatic efforts in this part of the world and other parts of the world where the Russian narrative resonates.\u201d Kyiv could expect that \u201cthis is the beginning of a larger shift in perception that could eventually translate into potential support,\u201d Borshchevskaya said. Similarly, the Ukrainian president\u2019s participation in the G7 summit is \u201ca message to the rest of the world, to Russia and beyond, and the so-called Global South,\u201d Gould-Davies believes. There is a concern in the West over the extent to which some major developing economies \u2013 Brazil, South Africa and, to a degree, India \u2013 \u201care not criticizing, not condemning Russia and indeed in various ways are helping to mitigate the impact of sanctions on Russia,\u201d he said. \u201cCollectively, economically, they matter. So there is, I think, this felt need for a renewed diplomatic campaign to bring some of these most important states into the kind of the Western way of looking at these things,\u201d Gould-Davies said. ___ Associated Press writers Danica Kirka in London and Gerald Imray in Cape Town, South Africa, contributed. ___ Follow AP's coverage of the war in Ukraine at https://apnews.com/hub/russia-ukraine Syrian President Bashar al-Assad (L) is welcomed in Jeddah on the eve of the Arab League summit Ukrainian President Volodymyr Zelensky said he landed Friday in Saudi Arabia, host of an Arab League summit attended by long isolated Syrian President Bashar al-Assad, a close Russian ally. The previously unannounced visit is Zelensky's first to the Middle East since Moscow's invasion in February 2022, giving the Ukrainian leader an opportunity to address leaders in the region that has been far less united in its support of Kyiv than staunch Western allies. \"Arrived in Saudi Arabia. I will speak at the Arab League summit,\" Zelensky said on Twitter, adding he plans to meet with Saudi Crown Prince Mohammed bin Salman and other leaders. He arrived in the Red Sea coastal city of Jeddah one day after Assad, whose government is being readmitted to the Arab League after its suspension in 2011 over the brutal crackdown on pro-democracy demonstrators that led to civil war. The summit in Saudi Arabia comes at a time when the world's biggest oil exporter is flexing its diplomatic muscle across the Middle East and beyond. An Arab League official told AFP Zelenky's invitation came from Saudi Arabia, not the bloc.",
"target_text": "Ukranian President Zelenskyy attends Arab League summit in Saudi Arabia"
}
}
Any help would be greatly appreciated!
Based on your other question on this problem, where you mentioned that you're new to PyTorch, my answer reflects the general approach I personally use when I need to work with a new machine-learning library/algorithm I am unfamiliar with.
Since T5 is a popular algorithm, I tried to find a "template" showing how to fine-tune it for summarization, where my code represents an adaptation of this source, which I found by reading the documentation on T5 of the transformers library. Most importantly, I modified the CustomDataset class so it's compatible with the data you're working with; also, I modified some pieces of the code which returned warnings. Please note that the approach uses PyTorch for training but loads T5 using transformers.
As shown below, the example code trains T5 "successfully" in terms of obtaining a lower loss over time. Of course you still need to adapt parts of the code (such as increasing max_sequence_size to your requirement, moving the computations to GPU, add the validation code, etc); nevertheless, I'm sure you will be able to take it from here using the example below, the aforementioned source, as well as the official user guides of PyTorch and Transformers!
from transformers import T5Tokenizer, T5ForConditionalGeneration
import torch
from torch.utils.data import Dataset, DataLoader
input_data ={
"0": {
"input_text": "Ukrainian President Volodymyr Zelenskiy attended a summit of the Arab League in Saudi Arabia on Friday to canvas support for his people, while Saudi Crown Prince Mohammed bin Salman expressed his readiness to mediate in the war between Moscow and Kyiv. Also at the Jeddah gathering, Arab leaders warmly welcomed back into their fold Syria\u2019s President Bashar al-Assad \u2014 who has received heavy support from Russia in his country\u2019s civil war \u2014 following a decade of isolation. \u201cWe reaffirm the kingdom\u2019s readiness to continue mediating efforts between Russia and Ukraine, and to support all international efforts aimed at resolving the crisis politically in a way that contributes to achieving security,\u201d the Saudi Crown Prince said in his opening speech. Prince Mohammed has mediated in the conflict before. Zelenskiy, who was also due to attend a summit of the G7 leaders in the Japanese city of Hiroshima this weekend, thanked Saudi Arabia for its past help and said delegates would each receive the text of his 10-point peace plan. He asked them to work with Ukraine directly without intermediaries. Gulf states have tried to remain neutral in the Ukraine conflict despite Western pressure on Gulf oil producers to help isolate Russia, a fellow OPEC+ member. Saving people In his address to the summit, Zelenskiy said some countries including members of the Arab League preferred to \u201cturn a blind eye\u201d to Russia\u2019s illegal annexation of Ukrainian land and to its jailing of some Ukrainians during the 15-month war. \u201cI am sure we can all be united in saving people from the cages of Russian prisons,\u201d he said, speaking in English. Last year, in a diplomatic coup, Crown Prince Mohammed secured the release of 10 foreigners captured by Russia in Ukraine. The move was apparently made possible by his close ties with Russian President Vladimir Putin. \u201cThe Kingdom of Saudi Arabia plays a significant role and we are ready to take our cooperation to a new level,\u201d Zelenskiy said wrote on Twitter shortly after arriving in Jeddah. Saudi Arabia faced heavy criticism from the United States over an OPEC+ decision to cut oil production, seen as helping Russia to refill its coffers by boosting prices. Even though the October decision initially drew the ire of the United States and other Western countries, market dynamics since then have shown the cuts to be prudent. At a time when Russia\u2019s war on Ukraine has roiled global energy markets, the role the kingdom plays as the world\u2019s largest oil exporter has grown in importance to both Washington and Moscow. KYIV, Ukraine (AP) \u2014 Ukrainian President Volodymyr Zelenskyy addressed a summit of Arab leaders in Saudi Arabia on Friday before what a senior official said would be a trip to Japan for a meeting with the leaders of the world\u2019s most powerful democracies. Zelenskyy has in recent months made foreign trips to shore up diplomatic support for Ukraine\u2019s fight against Russia\u2019s full-scale invasion almost 15 months ago and solicit more military support. He earlier this week returned from a three-day trip to Italy, the Vatican, Germany, France and the United Kingdom. Ukraine and Russia are squaring up for a major and potentially decisive phase of the war as Kyiv prepares an expected counteroffensive. The conflict has been bogged down in a war of attrition in recent months amid bad weather. Zelenskyy\u2019s office said he was invited to attend the Arab League summit in Jeddah, where he met with Saudi Crown Prince Mohammed bin Salman before holding other bilateral meetings. They discussed Zelenskyy\u2019s peace plan, the security situation in Ukraine and possible investments in the reconstruction of the country, a presidential statement said. Zelenskyy also invited Prince Mohammed to visit Ukraine. Zelenskyy urged leaders at the summit to resist Moscow\u2019s influence and consider his peace proposals, which include the withdrawal of the Kremlin\u2019s forces from occupied areas of Ukraine. \u201cI\u2019m more than sure that none of you will agree to surrender a third of your country to the invaders,\u201d Zelenskyy said in English. \u201cAnother priority is the protection of the Muslim community of Ukraine,\u201d Zelenskyy said. \u201cCrimea was the first to suffer from the Russian occupation, and most of those who suffer repression in occupied Crimea are Muslims.\u201d Crimean Tatar leader Mustafa Dzhemilev accompanied Zelenskyy on the visit. Zelenskyy will later travel to a Group of Seven summit in Japan, where leaders of the world\u2019s most powerful democracies aim to step up punishment on Russia for its full-scale invasion of Ukraine, according to Oleksiy Danilov, the secretary of Ukraine\u2019s National Security and Defense Council. However, Danilov\u2019s office later posted a statement backtracking on his announcement and saying Zelenskyy would appear at the G-7 summit via video. Zelenskyy\u2019s movements are kept secret for security reasons. Meanwhile, Russian forces kept up their long-range bombardment of Ukrainian targets while drones reportedly damaged train lines behind their front line. About 130 meters (430 feet) of railway track were damaged and trains were halted for hours after an explosion derailed eight cars of a freight train carrying grain in Russia-occupied Crimea, Russian state media reported Friday. Thursday\u2019s blast prompted renewed suspicions about possible Ukrainian saboteur activity behind Russian lines. Train traffic was also halted in northern Crimea on Thursday night after a drone hit a railway track near the town of Dzhankoi, Russia\u2019s Baza Telegram channel reported. Sergei Aksyonov, the Kremlin-appointed head of Crimea, said in a separate post that four Ukrainian drones were shot down overnight in the peninsula\u2019s north. Aksyonov claimed there was no damage or casualties. Russia overnight fired cruise missiles, drones and artillery at targets across Ukraine, killing two civilians, officials said Friday. The attacks included an air assault on Kyiv for the second straight day and the 10th time in three weeks. The Kremlin\u2019s forces also took aim at central, eastern and southern Ukraine, and the western Lviv region near the border with Poland. Russia launched 22 Iranian-made Shahed drones and six Kalibr cruise missiles during the night, the Ukrainian Air Force said. It said air defenses downed 16 drones and three missiles. The Russian shelling killed two civilians and wounded nine others in Ukraine\u2019s eastern Donetsk region, said its governor, Pavlo Kyrylenko. The missile attacks that have intensified recently aim to \u201cdisrupt Ukraine\u2019s plans and preparations for active military operations during the spring-summer campaign,\u201d according to a statement from Ukraine\u2019s intelligence agency, published on Telegram. The targets are Ukraine\u2019s military control points and barracks, supply routes and the places where ammunition, equipment, fuel are stored, it said. On Friday, the United Nations said operations to ship Ukrainian grain were \u201cpartially restarting,\u201d two days after Russia gave a green light to extend the deal for two months. The U.N. also urged a swift return to the previous tempo of ship arrivals and departures from all three Black Sea ports and inspections of their cargo. U.N. associate spokesperson Stephanie Tremblay said the Joint Coordination Center, which includes representatives from the four parties involved in the deal \u2013 Russia, Ukraine, Turkey and the United Nations -- approved the registration Friday of six new vessels to participate in the grain shipments. Nine applications to participate remain pending, she said. No ships are currently loading at any of the three ports, Tremblay said, but inspection teams from the center checked and cleared three new vessels Friday to proceed to the ports of Odesa and Chornomorsk. ___ Hanna Arhirova in Kyiv and Edith M. Lederer at the United Nations contributed to this report. ___ Follow AP\u2019s coverage of the war in Ukraine at https://apnews.com/hub/russia-ukraine President Zelenskyy makes a stop in Saudi Arabia on his way to Japan to meet with the G7. At the G7 increased sanctions against Russia are on the table, but negotiations still continue. A look at the battle in Bakhmut plus Crimean Tartars; expelled by the Soviets decades ago, now looking to return. TALLINN, Estonia (AP) \u2014 While the world awaits Ukraine\u2019s spring battlefield offensive, its leader, Volodymyr Zelenskyy, has launched a diplomatic one. In the span of a week, he's dashed to Italy, the Vatican, Germany, France and Britain to shore up support for defending his country. On Friday, he was in Saudi Arabia to meet with Arab leaders, some of whom are allies with Moscow. President Vladimir Putin, meanwhile, was in the southern Russian city of Pyatigorsk, chairing a meeting with local officials, sitting at a large table at a distance from the other attendees. The Russian president has faced unprecedented international isolation, with an International Criminal Court arrest warrant hanging over his head and clouding the prospects of traveling to many destinations, including those viewed as Moscow's allies. With his invasion of Ukraine, \u201cPutin took a gamble and lost really, really big time,\u201d said Theresa Fallon, director of the Brussels-based Centre for Russia Europe Asia Studies. \u201cHe is an international pariah, really.\u201d It was only 10 years ago when Putin stood proudly among his peers at the time -\u2013 Barack Obama, Angela Merkel and Shinzo Abe \u2013 at a Group of Eight summit in Northern Ireland. Russia has since been kicked out of the group, which consists of Canada, France, Germany, Italy, Japan, Britain and the United States, for illegally annexing Crimea in 2014. Now it appears to be Ukraine\u2019s turn in the spotlight. There were conflicting messages from Kyiv whether Zelenskyy would attend the G7 in Japan on Sunday. The secretary of Ukraine\u2019s National Security and Defense Council said on national television the president would be there, but the council later walked back those remarks, saying Zelenskyy would join via video link. The president\u2019s office would not confirm either way for security reasons. But whether in person or via video, it would be of great symbolic and geopolitical significance. \u201cIt conveys the fact that the G7 continues to strongly support Ukraine,\u201d said Nigel Gould-Davies, senior fellow for Russia and Eurasia at the International Institute for Strategic Studies. \u201cIt\u2019s a visible marker of the continued commitment of the most highly industrialized and highly developed countries in the world.\u201d Story continues It also comes at a time when the optics are just not in the Kremlin\u2019s favor. There\u2019s uncertainty over whether Putin can travel to South Africa in August for a summit of the BRICS nations of Brazil, Russia, India, China and South Africa. Moscow has long showcased the alliance as an alternative to the West\u2019s global dominance, but this year it is already proving awkward for the Kremlin. South Africa, the host of the summit, is a signatory to the ICC and is obligated to comply with the arrest warrant on war crimes charges. South Africa has not announced that Putin will definitely come to the summit but has been planning for his possible arrival. South African President Cyril Ramaphosa has appointed an inter-ministerial committee, led by Deputy President Paul Mashatile, to consider South Africa\u2019s options with regard to its ICC commitment over Putin\u2019s possible trip. While it is highly unlikely the Russian president would be arrested there if he decides to go, the public debate about whether he can is in itself \u201can unwelcome development whose impact should not be underestimated,\u201d according to Gould-Davies. Then there are Moscow\u2019s complicated relations with its own neighbors. Ten days ago, Putin projected the image of solidarity, with leaders of Armenia, Belarus and Central Asian states standing beside him at a Victory Day military parade on Red Square. This week, however, the leaders of Kazakhstan, Kyrgyzstan, Tajikistan, Turkmenistan and Uzbekistan flocked to China and met with leader Xi Jinping at a summit that highlighted the erosion of Russia\u2019s influence in the region as Beijing seeks to make economic inroads into Central Asia. Xi is using the opportunity \u201cof a weakened Russia, a distracted Russia, almost a pariah-state Russia to increase (China\u2019s) influence in the region,\u201d Fallon said. Putin\u2019s effort this month to shore up more friends in the South Caucasus by scrapping visa requirements for Georgian nationals and lifting a four-year ban on direct flights to the country also didn\u2019t appear to go as smoothly as the Kremlin may have hoped. The first flight that landed Friday in Georgia was met with protests, and the country\u2019s pro-Western president has decried the move as a provocation. Zelenskyy\u2019s ongoing world tour can be seen as a success on many levels. Invitations from other world leaders is a sign they think Ukraine is \"going to come out of the war in good shape,\u201d said Phillips P. O\u2019Brien, professor of strategic studies at the University of St. Andrews in Scotland. Otherwise, \u201cit simply wouldn\u2019t be happening,\u201d he said. \"No one would want to be around a leader they think is going to be defeated and a country that\u2019s going to collapse.\u201d By contrast, the ICC warrant might make it harder for leaders even to visit Putin in Moscow because \u201cit\u2019s not a good look to visit an indicted war criminal,\u201d Gould-Davies said. European leaders promised him an arsenal of missiles, tanks and drones, and even though no commitment has been made on fighter jets \u2013 something Kyiv has wanted for months \u2013 a conversation about finding ways to do it has begun. His appearance Friday at the Arab League summit in Jeddah, a Saudi Arabian port on the Red Sea, highlighted Kyiv\u2019s effort to spread its plight for support far and wide, including in some countries whose sympathies are with Russia. In addition to Zelenskyy, Saudi Crown Prince Mohammed bin Salman also welcomed Syrian President Bashar Assad at the summit after a 12-year suspension \u2013 something analysts say aligns with Moscow\u2019s interests. Anna Borshchevskaya, a senior fellow at the Washington Institute who focuses on Russia\u2019s policy in the Middle East, called it \u201canother testament to the fact that Russia is not isolated globally for its invasion of Ukraine, that the Middle East is one part of the world where Russia is able to find avenues to avoid global isolation \u2013 both ideological isolation but also economic isolation.\u201d She added that Zelenskyy and his government deserve credit for \u201cin recognizing that they need to reach out more to improve their diplomatic efforts in this part of the world and other parts of the world where the Russian narrative resonates.\u201d Kyiv could expect that \u201cthis is the beginning of a larger shift in perception that could eventually translate into potential support,\u201d Borshchevskaya said. Similarly, the Ukrainian president\u2019s participation in the G7 summit is \u201ca message to the rest of the world, to Russia and beyond, and the so-called Global South,\u201d Gould-Davies believes. There is a concern in the West over the extent to which some major developing economies \u2013 Brazil, South Africa and, to a degree, India \u2013 \u201care not criticizing, not condemning Russia and indeed in various ways are helping to mitigate the impact of sanctions on Russia,\u201d he said. \u201cCollectively, economically, they matter. So there is, I think, this felt need for a renewed diplomatic campaign to bring some of these most important states into the kind of the Western way of looking at these things,\u201d Gould-Davies said. ___ Associated Press writers Danica Kirka in London and Gerald Imray in Cape Town, South Africa, contributed. ___ Follow AP's coverage of the war in Ukraine at https://apnews.com/hub/russia-ukraine Syrian President Bashar al-Assad (L) is welcomed in Jeddah on the eve of the Arab League summit Ukrainian President Volodymyr Zelensky said he landed Friday in Saudi Arabia, host of an Arab League summit attended by long isolated Syrian President Bashar al-Assad, a close Russian ally. The previously unannounced visit is Zelensky's first to the Middle East since Moscow's invasion in February 2022, giving the Ukrainian leader an opportunity to address leaders in the region that has been far less united in its support of Kyiv than staunch Western allies. \"Arrived in Saudi Arabia. I will speak at the Arab League summit,\" Zelensky said on Twitter, adding he plans to meet with Saudi Crown Prince Mohammed bin Salman and other leaders. He arrived in the Red Sea coastal city of Jeddah one day after Assad, whose government is being readmitted to the Arab League after its suspension in 2011 over the brutal crackdown on pro-democracy demonstrators that led to civil war. The summit in Saudi Arabia comes at a time when the world's biggest oil exporter is flexing its diplomatic muscle across the Middle East and beyond. An Arab League official told AFP Zelenky's invitation came from Saudi Arabia, not the bloc.",
"target_text": "Ukranian President Zelenskyy attends Arab League summit in Saudi Arabia"
}
}
# duplicate single entry to get some additional "examples"
for i in range(1, 5):
input_data[str(i)] = dict()
input_data[str(i)]['input_text'] = input_data['0']['input_text']
input_data[str(i)]['target_text'] = input_data['0']['target_text']
class CustomDataset(Dataset):
def __init__(self, data_dict, tokenizer, max_sequence_size, max_summary_len):
self.tokenizer = tokenizer
self.data_dict = data_dict
self.max_sequence_size = max_sequence_size
self.max_summary_len = max_summary_len
self.input_text = [
"summarize: " + self.data_dict[key]['input_text']
for key in self.data_dict.keys()
]
self.target_text = [
self.data_dict[key]['target_text']
for key in self.data_dict.keys()
]
def __len__(self):
return len(self.input_text)
def __getitem__(self, index):
input_text_at_index = self.input_text[index]
target_text_at_index = self.target_text[index]
source = self.tokenizer.batch_encode_plus(
batch_text_or_text_pairs=[input_text_at_index],
max_length= self.max_sequence_size,
padding='max_length',
return_tensors='pt',
truncation=True
)
target = self.tokenizer.batch_encode_plus(
batch_text_or_text_pairs=[target_text_at_index],
max_length= self.max_summary_len,
padding='max_length',
return_tensors='pt',
truncation=True
)
source_ids = source['input_ids'].squeeze()
source_mask = source['attention_mask'].squeeze()
target_ids = target['input_ids'].squeeze()
target_mask = target['attention_mask'].squeeze()
return {
'source_ids': source_ids.to(dtype=torch.long),
'source_mask': source_mask.to(dtype=torch.long),
'target_ids': target_ids.to(dtype=torch.long),
'target_ids_y': target_ids.to(dtype=torch.long)
}
def train(epoch, tokenizer, model, loader, optimizer):
model.train()
for i, data in enumerate(loader, 0):
y = data['target_ids'] #.to(device, dtype = torch.long)
y_ids = y[:, :-1].contiguous()
lm_labels = y[:, 1:].clone().detach()
lm_labels[y[:, 1:] == tokenizer.pad_token_id] = -100
ids = data['source_ids'] #.to(device, dtype = torch.long)
mask = data['source_mask'] #.to(device, dtype = torch.long)
outputs = model(input_ids = ids, attention_mask = mask, decoder_input_ids=y_ids, labels=lm_labels)
loss = outputs[0]
print(f'Epoch: {epoch}, Loss: {loss.item()}')
optimizer.zero_grad()
loss.backward()
optimizer.step()
MAX_SEQUENCE_SIZE = 600
tokenizer = T5Tokenizer.from_pretrained(
"t5-base",
model_max_length=MAX_SEQUENCE_SIZE
)
model = T5ForConditionalGeneration.from_pretrained("t5-base")
dataset = CustomDataset(
data_dict = input_data,
tokenizer = tokenizer,
max_sequence_size=MAX_SEQUENCE_SIZE,
max_summary_len=128
)
# Defining the parameters for creation of dataloaders
train_params = {
'batch_size': 2,
'shuffle': True,
'num_workers': 0
}
training_loader = DataLoader(dataset, **train_params)
optimizer = torch.optim.Adam(
params=model.parameters(),
lr=0.001
)
for epoch in range(2):
train(
epoch=epoch,
tokenizer=tokenizer,
model=model,
loader=training_loader,
optimizer=optimizer
)
# Epoch: 0, Loss: 6.379357814788818
# Epoch: 0, Loss: 2.551377534866333
# Epoch: 0, Loss: 2.4488468170166016
# Epoch: 1, Loss: 1.555290937423706
# Epoch: 1, Loss: 0.6974927186965942
# Epoch: 1, Loss: 0.1120450347661972