Below are the details of the upgrades we performed :
| MarkLogic | DHF | Gradle | mlcp/XCC | Java Client API | ml-javaclient-util | |
|---|---|---|---|---|---|---|
| Before | 10.0-9.2 | 5.2.0 | 5.2.1 | 9.0.8/9.0.7 | 4.2.0 | 3.13.3 |
| After | 11.1.0 | 5.8.1 | 6.4 | 11.0.3 | 6.3.0 | 4.6.0 |
Post the upgrade we have encountered a persisting issue with the JOBS database. Currently, it fails to capture complete step details. Specifically, it only retains information for the step currently executing, overwriting the data of the previous step as soon as the subsequent step begins.
It is working fine in the environments where the upgrades have not yet been done.
Below are the to sample provenance files:
"job": {
"jobId": "d109d96c-9020-44f0-8f84-5fb711eb1420/2023-28/2024-01-18T18:53:25.865",
"flow": "*********",
"user": "*********",
"lastAttemptedStep": "14",
"lastCompletedStep": "14",
"jobStatus": "finished",
"timeStarted": "2024-01-18T22:24:47.394379+01:00",
"stepResponses": {
"14": {
"flowName": "*********",
"stepName": "*********",
"stepDefinitionName": "*********",
"stepDefinitionType": "custom",
"targetEntityType": "MatchingResult",
"targetDatabase": "***-data-hub-STAGING",
"stepOutput": null,
"fullOutput": null,
"status": "completed step 14",
"totalEvents": 1,
"successfulEvents": 1,
"failedEvents": 0,
"successfulBatches": 1,
"failedBatches": 0,
"success": true,
"stepStartTime": "2024-01-18T22:24:47.423718+01:00",
"stepEndTime": "2024-01-18T22:27:27.983826+01:00"
}
},
"timeEnded": "2024-01-18T22:27:28.018498+01:00"
}
}```
"job": {
"jobId": "6d3be8aa-062f-4d6d-8d56-d313ca18b1c9/2023-24/2023-06-26T08:52:24.884",
"flow": "*********",
"user": "*********",
"lastAttemptedStep": "14",
"lastCompletedStep": "14",
"jobStatus": "finished",
"timeStarted": "2023-06-26T10:52:27.087495+02:00",
"timeEnded": "2023-06-26T11:25:36.168996+02:00",
"stepResponses": {
"1": {
"flowName": "*********",
"stepName": "*********",
"stepDefinitionName": "*********",
"stepDefinitionType": "custom",
"stepOutput": null,
"fullOutput": null,
"status": "completed step 1",
"totalEvents": 1,
"successfulEvents": 1,
"failedEvents": 0,
"successfulBatches": 1,
"failedBatches": 0,
"success": true,
"stepStartTime": "2023-06-26T10:52:27.13748+02:00",
"stepEndTime": "2023-06-26T10:52:32.374773+02:00"
},
"2": {
"flowName": "*********",
"stepName": "*********",
"stepDefinitionName": "*********",
"stepDefinitionType": "custom",
"stepOutput": null,
"fullOutput": null,
"status": "completed step 2",
"totalEvents": 1,
"successfulEvents": 1,
"failedEvents": 0,
"successfulBatches": 1,
"failedBatches": 0,
"success": true,
"stepStartTime": "2023-06-26T10:52:32.505048+02:00",
"stepEndTime": "2023-06-26T10:52:55.896385+02:00"
},
"3": {
"flowName": "*********",
"stepName": "*********",
"stepDefinitionName": "*********",
"stepDefinitionType": "custom",
"stepOutput": null,
"fullOutput": null,
"status": "completed step 3",
"totalEvents": 1,
"successfulEvents": 1,
"failedEvents": 0,
"successfulBatches": 1,
"failedBatches": 0,
"success": true,
"stepStartTime": "2023-06-26T10:52:56.025055+02:00",
"stepEndTime": "2023-06-26T10:53:13.547117+02:00"
},
"4": {
"flowName": "*********",
"stepName": "*********",
"stepDefinitionName": "*********",
"stepDefinitionType": "custom",
"stepOutput": null,
"fullOutput": null,
"status": "completed step 4",
"totalEvents": 2497,
"successfulEvents": 22497,
"failedEvents": 0,
"successfulBatches": 225,
"failedBatches": 0,
"success": true,
"stepStartTime": "2023-06-26T10:53:13.683279+02:00",
"stepEndTime": "2023-06-26T11:10:02.097445+02:00"
},
"5": {
"flowName": "*********",
"stepName": "*********",
"stepDefinitionName": "*********",
"stepDefinitionType": "custom",
"stepOutput": null,
"fullOutput": null,
"status": "completed step 5",
"totalEvents": 1,
"successfulEvents": 1,
"failedEvents": 0,
"successfulBatches": 1,
"failedBatches": 0,
"success": true,
"stepStartTime": "2023-06-26T11:10:02.894001+02:00",
"stepEndTime": "2023-06-26T11:10:25.212723+02:00"
},
"6": {
"flowName": "*********",
"stepName": "*********",
"stepDefinitionName": "*********",
"stepDefinitionType": "custom",
"stepOutput": null,
"fullOutput": null,
"status": "completed step 6",
"totalEvents": 1,
"successfulEvents": 1,
"failedEvents": 0,
"successfulBatches": 1,
"failedBatches": 0,
"success": true,
"stepStartTime": "2023-06-26T11:10:25.335696+02:00",
"stepEndTime": "2023-06-26T11:10:25.564489+02:00"
},
"7": {
"flowName": "*********",
"stepName": "*********",
"stepDefinitionName": "*********",
"stepDefinitionType": "custom",
"stepOutput": null,
"fullOutput": null,
"status": "completed step 7",
"totalEvents": 11,
"successfulEvents": 15011,
"failedEvents": 0,
"successfulBatches": 151,
"failedBatches": 0,
"success": true,
"stepStartTime": "2023-06-26T11:10:25.650559+02:00",
"stepEndTime": "2023-06-26T11:18:10.419283+02:00"
},
"8": {
"flowName": "*********",
"stepName": "*********",
"stepDefinitionName": "*********",
"stepDefinitionType": "custom",
"stepOutput": null,
"fullOutput": null,
"status": "completed step 8",
"totalEvents": 1,
"successfulEvents": 1,
"failedEvents": 0,
"successfulBatches": 1,
"failedBatches": 0,
"success": true,
"stepStartTime": "2023-06-26T11:18:11.105034+02:00",
"stepEndTime": "2023-06-26T11:19:04.197125+02:00"
},
"9": {
"flowName": "*********",
"stepName": "*********",
"stepDefinitionName": "*********",
"stepDefinitionType": "custom",
"stepOutput": null,
"fullOutput": null,
"status": "completed step 9",
"totalEvents": 0,
"successfulEvents": 0,
"failedEvents": 0,
"successfulBatches": 0,
"failedBatches": 0,
"success": true,
"stepStartTime": "2023-06-26T11:19:04.310099+02:00",
"stepEndTime": "2023-06-26T11:19:04.416497+02:00"
},
"10": {
"flowName": "*********",
"stepName": "*********",
"stepDefinitionName": "*********",
"stepDefinitionType": "custom",
"stepOutput": null,
"fullOutput": null,
"status": "completed step 10",
"totalEvents": 0,
"successfulEvents": 0,
"failedEvents": 0,
"successfulBatches": 0,
"failedBatches": 0,
"success": true,
"stepStartTime": "2023-06-26T11:19:04.567946+02:00",
"stepEndTime": "2023-06-26T11:19:04.64965+02:00"
},
"11": {
"flowName": "*********",
"stepName": "*********",
"stepDefinitionName": "*********",
"stepDefinitionType": "custom",
"stepOutput": null,
"fullOutput": null,
"status": "completed step 11",
"totalEvents": 0,
"successfulEvents": 0,
"failedEvents": 0,
"successfulBatches": 0,
"failedBatches": 0,
"success": true,
"stepStartTime": "2023-06-26T11:19:04.868482+02:00",
"stepEndTime": "2023-06-26T11:19:05.022998+02:00"
},
"12": {
"flowName": "*********",
"stepName": "*********",
"stepDefinitionName": "*********",
"stepDefinitionType": "custom",
"stepOutput": null,
"fullOutput": null,
"status": "completed step 12",
"totalEvents": 1,
"successfulEvents": 1,
"failedEvents": 0,
"successfulBatches": 1,
"failedBatches": 0,
"success": true,
"stepStartTime": "2023-06-26T11:19:05.179518+02:00",
"stepEndTime": "2023-06-26T11:19:56.693317+02:00"
},
"13": {
"flowName": "*********",
"stepName": "*********",
"stepDefinitionName": "*********",
"stepDefinitionType": "custom",
"stepOutput": null,
"fullOutput": null,
"status": "completed step 13",
"totalEvents": 635,
"successfulEvents": 635,
"failedEvents": 0,
"successfulBatches": 7,
"failedBatches": 0,
"success": true,
"stepStartTime": "2023-06-26T11:19:56.816666+02:00",
"stepEndTime": "2023-06-26T11:22:04.559349+02:00"
},
"14": {
"flowName": "*********",
"stepName": "*********",
"stepDefinitionName": "*********",
"stepDefinitionType": "custom",
"stepOutput": null,
"fullOutput": null,
"status": "completed step 14",
"totalEvents": 1,
"successfulEvents": 1,
"failedEvents": 0,
"successfulBatches": 1,
"failedBatches": 0,
"success": true,
"stepStartTime": "2023-06-26T11:22:04.716391+02:00",
"stepEndTime": "2023-06-26T11:25:36.095889+02:00"
}
}
}
}```
Steps are functioning correctly. Additionally, post-DHF upgrade, we noticed new modules introduced, including:
/data-hub/5/data-services/job/startJob.sjs
/data-hub/5/data-services/job/startStep.sjs
/data-hub/5/data-services/job/finishStep.sjs
/data-hub/5/data-services/job/finishJob.sjs
/data-hub/5/data-services/job/getJob.sjs
Do any modifications need to be made to these modules?
Closing this as MarkLogic Support has confirmed that this is expect behaviour.