diff --git a/.build/cspell-words.txt b/.build/cspell-words.txt index 5030b98410..554b717140 100644 --- a/.build/cspell-words.txt +++ b/.build/cspell-words.txt @@ -1,8 +1,8 @@ adfs Adsi amsi -Antispam anob +Antispam asmx authsspi autodiscover @@ -26,8 +26,8 @@ dumptidset DWORD eems EFORMS -EICAR eicar +EICAR Emotet emsmdb Entra @@ -37,12 +37,13 @@ Eventlog evtx Exchweb exfiltration +EXOMTL fabrikam FIPFS fips -Fsis fltmc freb +Fsis FYDIBOHF GCDO Get-AuthenticodeSignature @@ -92,8 +93,8 @@ Mgmt mitigations msdcs MSDTC -MSERT msert +MSERT msipc msrc Multiplexor @@ -109,8 +110,8 @@ notin notlike notmatch nspi -ntlm NTFS +ntlm NUMA nupkg odata @@ -138,7 +139,6 @@ Runtimes sccm Schannel SCOM -!scriptblock SERVERNAME Servicehost servicelet diff --git a/Transport/Get-EXOMTLReport.ps1 b/Transport/Get-EXOMTLReport.ps1 new file mode 100644 index 0000000000..5634d3e688 --- /dev/null +++ b/Transport/Get-EXOMTLReport.ps1 @@ -0,0 +1,351 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# +.NOTES + Name: Get-EXOMTLReport.ps1 + Requires: User Rights + +.SYNOPSIS +Reads thru an EXO sourced Message Tracking log to generate plain text reporting on what is in the log. + +.DESCRIPTION +Reads Message Tracking Detailed logs from EXO to generate reporting on critical information that they contain. +Start-HistoricalSearch -ReportTitle -StartDate <24 hours before sent> -EndDate <24 hours after sent> -ReportType MessageTraceDetail -MessageID <message ID> -NotifyAddress <address to notify> + +Parses and provides details about the message in the MTL. +Helpful in troubleshooting message delivery issues. + +.PARAMETER MTLFile +MTL File to process. + +.PARAMETER ReportPath +Folder path for the output file. + +.PARAMETER MessageID +MessageID of a message to parse if there is more than one in the MTL. + +.OUTPUTS +Text File broken into sections that contain the output of the various data gathering run against the MTL. + +Default Output File: +$PSScriptRoot\MTL_Report_<date>.txt + +.EXAMPLE +.\Get-EXOMTLReport.ps1 -MTLPath C:\temp\MyMtl.csv -MessageID <123214124@myserver.com> + +Generates a report from the MyMtl.csv file of the message with ID <123214124@myserver.com> + +#> + +[CmdletBinding()] +param ( + [Parameter(Mandatory = $true)] + [string] + $MTLFile, + [Parameter()] + [string] + $ReportPath = $PSScriptRoot, + [Parameter()] + [string] + $MessageID, + [Parameter()] + [bool] + $SkipUpdate = $false +) + +. $PSScriptRoot\..\Shared\ScriptUpdateFunctions\Test-ScriptVersion.ps1 + +### Utilities ### +function Import-MTL { + [CmdletBinding()] + [OutputType([array])] + param ( + # File path for MTL to import + [Parameter(Mandatory = $true)] + [string] + $FilePath + ) + + # Test the path of the MTL + if (!(Test-Path $FilePath)) { + Write-Error "Unable to find the specified file" -ErrorAction Stop + } + + # Try to load the file with Unicode since we need to start somewhere. + $initial_mtl = Import-Csv $FilePath -Encoding Unicode + + # If it is null then we need to try without Unicode + if ($null -eq $initial_mtl) { + Write-Information "Failed to Load as Unicode; trying normal load" + $initial_mtl = Import-Csv $FilePath + # If we still have nothing then log an error and fail + if ($null -eq $initial_mtl) { + Write-Error "Failed to load CSV" -ErrorAction Stop + } + # Need to know that we loaded without Unicode. + else { + Write-Information "Loaded CSV without Unicode" + } + } else { + Write-Information "Loaded MTL with Unicode" + } + + # Making sure the MTL contains the fields we want. + if (!(Test-CSVData -CSV $initial_mtl -ColumnsToCheck "date_time_utc", "source_context", "connector_id", "source", "event_id", "message_id", "recipient_address", "recipient_status", "recipient_count", "related_recipient_address", "reference", "message_subject", "sender_address", "return_path", "message_info", "directionality", "custom_data")) { + Write-Error "MTL is missing one or more required fields." -ErrorAction Stop + } else { Write-Information "CSV Passed Validation" } + + # Converting our strings into [DateTime] + Write-Information "Converting date_time_utc values" + for ($i = 0; $i -lt $initial_mtl.Count; $i++) { + try { + $initial_mtl[$i].date_time_utc = Get-Date($initial_mtl[$i].date_time_utc) + } catch { + Write-Error ("Problem converting date information: " + $Error) -ErrorAction Stop + } + } + + return $initial_mtl +} + +# Gather up all of the entries related to a single MessageID +function Group-ByMessageID { + [CmdletBinding()] + [OutputType([array])] + param ( + # MTL array to process + [Parameter(Mandatory = $true)] + [array]$MTL, + # MessageID to group by + [Parameter(Mandatory = $true)] + [string]$MessageID + ) + + # Filter the MTL by our messageID + [array]$Output = $MTL | Where-Object { $_.message_id -eq $MessageID } + + # Make sure we found the messageID + if ($null -eq $Output) { + Write-Error ("MessageID " + $MessageID + " not found in provide MTL.") -ErrorAction Stop + } + + ### Do we want to search the reference Colum here as well?? + + return $Output +} + +# Test if we have only a single MessageID provided in the MTL +function Test-UniqueMessageID { + [CmdletBinding()] + [OutputType([bool])] + param ( + # Parameter help description + [Parameter(Mandatory = $true)] + [array] + $MTL + ) + + if (($MTL | Select-Object -Property message_id -Unique).count -gt 1) { + return $false + } else { + return $true + } +} + +# Makes sure that the provided CSV file has the needed columns to be a valid MTL +function Test-CSVData { + param( + [array]$CSV, + [array]$ColumnsToCheck + ) + + # Check to make sure we have data in the CSV + if (($null -eq $CSV) -or !($CSV.count -gt 0)) { + Write-Error "Provided CSV null or empty" -ErrorAction Stop + return $false + } + + # Read thru the data and make sure we have the needed columns + $ColumnHeaders = ($CSV | Get-Member -MemberType NoteProperty).Name.replace("`"", "") + foreach ( $ToCheck in $ColumnsToCheck) { + if (!($ColumnHeaders -contains $ToCheck)) { + # Write-Information ("Missing " + $ToCheck) + return $false + } + } + return $true +} + +function Write-InformationFile { + [CmdletBinding()] + param ( + # Parameter help description + [Parameter(Mandatory = $true)] + [string] + $header, + [Parameter(Mandatory = $false)] + [string] + $message, + [Parameter(Mandatory = $false)] + [System.Management.Automation.OrderedHashtable] + $myTable + ) + + $file = $ReportFile + + Add-Content -Path $file $header.ToUpper() + Add-Content -Path $file "====================" + $myTable | Format-Table -AutoSize -HideTableHeaders | Out-String | Add-Content -Path $file +} + +### Diagnostics ### + +# Determine and report the type of client that submitted the message +function Get-StoreSubmissionData { + [CmdletBinding()] + param ( + # Parameter help description + [Parameter(Mandatory = $true)] + [array] + $messageIDFilteredEvents + ) + + # Select the StoreDriver Submit event for this messageID + [array]$entry = $messageIDFilteredEvents | Where-Object { $_.source -eq "StoreDriver" -and $_.event_id -eq "RECEIVE" } + + # If we have more than one submission event that is a problem + if ($entry.count -gt 1) { Write-Warning "Detected multiple Submission events for the same message" } + + # We can have multiple SMTP RECEIVE events if they are using add on services + foreach ($event in $entry) { + # Extract the submission data + $submission = ConvertFrom-StringData ($event.source_context -replace ",", " `n") -Delimiter ":" + + # Build the reporting hashtable + $hash = [ordered]@{ + DateTimeUTC = $event.date_time_utc + ClientType = $submission.ClientType + CreationTime = $submission.CreationTime + SubmittingMailbox = $submission.Mailbox + MessageClass = $submission.MessageClass + } + + Write-InformationFile -header "Submission Information" -myTable $hash + } +} + +function Get-MIMEData { + [CmdletBinding()] + param ( + # Parameter help description + [Parameter(Mandatory = $true)] + [array] + $messageIDFilteredEvents + ) + + # Select the StoreDriver Submit event for this messageID + [array]$entry = $messageIDFilteredEvents | Where-Object { $_.source -eq "SMTP" -and $_.event_id -eq "RECEIVE" } + + # We can have multiple SMTP RECEIVE events if they are using add on services + foreach ($event in $entry) { + # If there is something wrong with the CSV we can end up with a null custom_data field, detect and skip. + if ([string]::IsNullOrEmpty($event.custom_data)) { + Write-Warning "Custom Data field Empty for SMTP RECEIVE event. Skipping" + } else { + $mimeData = (ConvertFrom-StringData ($event.custom_data -replace ";", " `n") -Delimiter "=")["S:MimeParts"].split("S:")[1].split("/") + + # Build the reporting hashtable + $hash = [ordered]@{ + DateTimeUTC = $event.date_time_utc + AttachmentCount = $mimeData[0] + EmbeddedAttachments = $mimeData[1] + NumberOfMimeParts = $mimeData[2] + EmailMessageType = $mimeData[3] + EmailMimeComplianceStatus = $mimeData[4] + } + + Write-InformationFile -header "Detected Mime Information on Submission" -myTable $hash + } + } +} + +function Get-MTLStatistics { + [CmdletBinding()] + param ( + # Parameter help description + [Parameter(Mandatory = $true)] + [array] + $messageIDFilteredEvents + ) + + # Sort the events by time. + $sortedEvents = $messageIDFilteredEvents | Sort-Object -Property "date_time_utc" + $storeReceiveEvents = $messageIDFilteredEvents | Where-Object { $_.source -eq "StoreDriver" -and $_.event_id -like "RECEIVE" } + $SMTPReceiveEvents = $messageIDFilteredEvents | Where-Object { $_.source -eq "SMTP" -and $_.event_id -like "RECEIVE" } + $deliveryEvents = $messageIDFilteredEvents | Where-Object { $_.event_id -like "DELIVER" } + $sendExternalEvents = $messageIDFilteredEvents | Where-Object { $_.event_id -like "SendExternal" } + $SMTPResubmitEvents = $messageIDFilteredEvents | Where-Object { $_.event_id -like "RESUBMIT" } + + $hash = [ordered]@{ + MessageID = $sortedEvents[0].message_id + FirstEvent = $sortedEvents[0].date_time_utc + LastEvent = $sortedEvents[-1].date_time_utc + StoreReceiveEvents = $storeReceiveEvents.count + SMTPReceiveEvents = $SMTPReceiveEvents.count + SMTPResubmitEvents = $SMTPResubmitEvents.count + DeliveryEvents = $deliveryEvents.count + SendExternalEvents = $sendExternalEvents.count + } + + Write-InformationFile -header "General MTL Statistics" -myTable $hash +} + +### Main ### + +# Set InformationPreference to allow Write-Information to be displayed to the screen. +$OriginalInformationPreference = $InformationPreference +$InformationPreference = 'Continue' + +if ($SkipUpdate) { Write-Information "Skipping Update" } +else { + # See if we have an updated version. + if (Test-ScriptVersion -AutoUpdate) { + # Update was downloaded, so stop here. + Write-Host "Script was updated. Please rerun the command." + return + } +} + +#Import the MTL file. +$MTL = Import-MTL -FilePath $MTLFile + +# Make sure the path for the output is good +if (!(Test-Path $ReportPath)) { + Write-Error ("Unable to find report path " + $ReportPath) -ErrorAction Stop +} else { + $ReportFile = (Join-Path -Path $ReportPath -ChildPath ("MTL_Report_" + (Get-Date -Format FileDateTime).ToString() + ".txt")) +} + +# If no messageID was provided make sure that there is only one in the MTL +if ([string]::IsNullOrEmpty($MessageID)) { + if (!(Test-UniqueMessageID -MTL $MTL)) { + Write-Error "Multiple MessageIDs detected in MTL please using -MessageID to specify the one to examine" -ErrorAction Stop + } else { + $MessageIDFilteredMTL = $MTL + } +} +# If a messageID was provided then filter based on it +else { + $MessageIDFilteredMTL = Group-ByMessageID -MTL $MTL -MessageID $MessageID +} + +# Run the set of tests that we want to run and generate the output. +Write-Information "Generating Reporting" +Get-MTLStatistics -messageIDFilteredEvents $MessageIDFilteredMTL +Get-SToreSubmissionData -messageIDFilteredEvents $MessageIDFilteredMTL +Get-MIMEData -messageIDFilteredEvents $MessageIDFilteredMTL +Write-Information $ReportFile + +# Set informationPreference back to the original setting. +$InformationPreference = $OriginalInformationPreference diff --git a/docs/Transport/Get-EXOMTLReport.md b/docs/Transport/Get-EXOMTLReport.md new file mode 100644 index 0000000000..2453c1b78e --- /dev/null +++ b/docs/Transport/Get-EXOMTLReport.md @@ -0,0 +1,55 @@ +# Get-EXOMTLReport +Download the latest release: [Get-EXOMTLReport](https://github.com/microsoft/CSS-Exchange/releases/latest/download/Get-EXOMTLReport.ps1) + +Provides information about email messages sent thru EXO by parsing a detailed message tracking log. + +## DESCRIPTION +Parses thru EXO Message Tracking log to extract detailed information about the message and present it in a more readable format. + +### Exchange Online +Recommend using [Start-HistoricalSearch](https://learn.microsoft.com/en-us/powershell/module/exchange/start-historicalsearch?view=exchange-ps) in EXO to gather a detailed Message Tracking Log for processing. + +``` PowerShell +Start-HistoricalSearch -ReportTitle "Fabrikam Search" -StartDate 8/10/2024 -EndDate 8/12/2024 -ReportType MessageTraceDetail -SenderAddress michelle@fabrikam.com -NotifyAddress chris@contoso.com +``` + +### Exchange On Premises +Does NOT work with Exchange On Premises message tracking logs. + +## PARAMETER + +**-MTLFile** + +CSV output of Message Tracking Log to process. + +**-ReportPath** + +Folder path for the output file. + +**-MessageID** + +Specifies the messageID to gather information about if there is more than one in the provided Message Tracking Log. + +## Outputs + +### Text File + +* Message Statistics +* Submission Information (from non-smtp client) +* Mime Data + +### Default Output File: +``` PowerShell +$PSScriptRoot\MTL_Report_<date>.txt +``` + +## EXAMPLE +``` PowerShell +.\Get-EXOMTLReport -MTLPath C:\temp\MyMtl.csv +``` +Generates a report to the default path from the file C:\Temp\MyMtl.csv. + +``` PowerShell +.\Measure-EmailDelayInMTL -MTLPath C:\temp\LargeMTL.csv -ReportPath C:\output -MessageID "<1231421231@server.contoso.com>" +``` +Generates a report to the c:\output directory from the file C:\Temp\LargeMTL.csv focusing on the MessageID <1231421231@server.contoso.com> diff --git a/mkdocs.yml b/mkdocs.yml index 5f8b662e94..ac7d783435 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -136,6 +136,7 @@ nav: - Compute-TopExoRecipientsFromMessageTrace: Transport/Compute-TopExoRecipientsFromMessageTrace.md - ReplayQueueDatabases: Transport/ReplayQueueDatabases.md - Measure-EmailDelayInMTL: Transport/Measure-EmailDelayInMTL.md + - Get-EXOMTLReport: Transport/Get-EXOMTLReport.md theme: name: 'material' features: