Set SCOM maintenance mode when all servers in a list are down

Azure 2 Minutes

I had a customer request where they had relatively consistent errors with a give network and they wanted to suppress alerting when they had this issue arise.

I wrote a PowerShell script that would ping the servers in a file and if all failed would set maintenance mode for a SCOM group of servers.

Below is the script contents.

Set-StrictMode -Version 2
#PS Example Script Here
#Example script ping servers on specific network from hardcoded list if all servers down then put servers in maint
# Mode. If servers respond then take group out of maint


function CreateMyEvent ($iEID, $sourceEvent, $param1,$param2,$param3,$param4,$param5,$param6,$param7)
        {
            #//Create Source with PS first.

            $test1 = " "
#            //10 below is event id, 1 is category.
            $info = "Information";
            $ei = New-Object System.Diagnostics.EventInstance($iEID,1);
	    $eventLogTest = New-Object System.Diagnostics.EventLog;
	$eventLogTest.Log = "Application";
            $eventLogTest.Source = $sourceEvent;
	    "Writing Eid: " + $iEID + " with parameters"
            $eventLogTest.WriteEvent($ei, @($param1,$param2,$param3,$param4,$param5,$param6,$param7,$test1))
	    "Wrote to event log"
        }


Function GroupMaintMode 
#($ScomServer, $GroupDisplayName, $DurationInMin, $Reason, $Comment)
(
[Parameter(Mandatory=$true)][string]$ScomServer,
[Parameter(Mandatory=$true)][string]$GroupDisplayName,
[Parameter(Mandatory=$true)][Int32]$DurationInMin,
[Parameter(Mandatory=$true)][string]$Reason,
[Parameter(Mandatory=$false)][string]$Comment
){
 
Import-Module OperationsManager
New-SCOMManagementGroupConnection -ComputerName $ScomServer
 
ForEach ($group in (Get-ScomGroup -DisplayName  $GroupDisplayName))
    {
   If ($group.InMaintenanceMode -eq $false)
         {
            $group.ScheduleMaintenanceMode([datetime]::Now.touniversaltime(), `
            ([datetime]::Now).addminutes($DurationInMin).touniversaltime(), `
 
             "$Reason", "$Comment" , "Recursive")
         }
    }
 
}

Function GroupMaintModeStop
#($ScomServer, $GroupDisplayName, $DurationInMin, $Reason, $Comment)
(
[Parameter(Mandatory=$true)][string]$ScomServer,
[Parameter(Mandatory=$true)][string]$GroupDisplayName,
[Parameter(Mandatory=$true)][Int32]$DurationInMin,
[Parameter(Mandatory=$true)][string]$Reason,
[Parameter(Mandatory=$false)][string]$Comment
){
 
Import-Module OperationsManager
New-SCOMManagementGroupConnection -ComputerName $ScomServer
 
ForEach ($group in (Get-ScomGroup -DisplayName  $GroupDisplayName))
    {
   If ($group.InMaintenanceMode -eq $true)
         {
             "Group is in maint mode, so take it out"
            $group.StopMaintenanceMode([datetime]::Now.touniversaltime(), [Microsoft.EnterpriseManagement.Common.TraversalDepth]::Recursive)
         }
    }
 
}
 
#Begin of main routine###############################

$sourceEvt = "SCOMNetOutage"
if (!(test-path ` HKLM:\SYSTEM\CurrentControlSet\Services\Eventlog\Application\$sourceEvt )) 
{new-eventlog -Logname Application -source $sourceEvt  -ErrorAction SilentlyContinue} 

Write-Eventlog -Logname "Application" -EntryType Information -EventID 31 -Source $sourceEvt -Message "Begin Script"

$error.clear()
$startD = Get-Date


#NOTE: your server or list of IP's should be in new line delimited file below, you can change this be editing line below
$info = get-content C:\SCOMNetCheck\ServerList.txt

foreach ($serverPing in $info) 
{


   $serverPing

   if ($serverPing -eq "")
   {
      "Skipping blank server in list"
      continue
   }

   $pingSuccess = $FALSE

   $PingOkMessage = $serverPing + ": Server test-connection succeeded"
#Start all jobs
$scriptblock =  { test-Connection -ComputerName "$args[0]" -Count 1 -Quiet }
#$scriptblock
    Start-Job -ScriptBlock { test-Connection -ComputerName $args[0] -Count 1 -Quiet }  -ArgumentList @($serverPing)
 }

#Wait for all jobs that are pinging addresses in file this is threaded to improve speed
Get-Job | Wait-Job
 
#Get all job results
$Output = Get-Job | Receive-Job #| Out-GridView Comand, Name

foreach ($res in $Output) {
"res: " + $res
if ($res){
   $pingSuccess = $TRUE

}
}



if ($pingSuccess)
{
     write-Host "Found Server that successfully pinged we need to stop maint mode now to Allow for alerting" -ForegroundColor Green
#log event that removing alert suppression
  CreateMyEvent 11 $sourceEvt "Found Server that successfully pinged we need to stop maint mode now to Allow for alerting" "" $serverPing "" "" "" ""
  GroupMaintModeStop -ScomServer "localhost" -GroupDisplayName "Maint Mode Network Group1" -DurationInMin 10  -Reason "ApplicationInstallation" -Comment "Suppress Alerting on network from PowerShell Custom Example script"

}
else
{
     write-Host "All Servers failed ping we need to set maint mode now to suppress alerting" -ForegroundColor Red
#log event indicating suppressing alerts. Should probably alert on this condition
  CreateMyEvent 13 $sourceEvt "All Servers failed ping we need to set maint mode now to suppress alerting" "" $serverPing "" "" "" ""

#Usage (calling the GroupMaintMode function)
GroupMaintMode -ScomServer "localhost" -GroupDisplayName "Maint Mode Network Group1" -DurationInMin 30  -Reason "ApplicationInstallation" -Comment "Suppress Alerting on network from PowerShell Custom Example script"

}

"start time:"
$startD
"End time: " 
Get-Date

Write-Eventlog -Logname "Application" -EntryType Information -EventID 37 -Source $sourceEvt -Message "End Script"
c:
Return 1

Author

Published