header
header Register : : Login header
header
divider
menuleft
menuright
submenu
left

[August 25th, 2008] Check the home page regarding PowerShell related news from a brand new sponsor: Idera

Readling Kiwi Syslog logfiles
Last Post 11 Dec 2007 02:19 PM by bsonposh. 2 Replies.
Printer Friendly
Sort:
PrevPrev NextNext
You are not authorized to post a reply.
Author Messages
JacobSaabyUser is Offline
New Member
New Member
Posts:17
Avatar

--
07 Dec 2007 07:45 AM  
Hey all,

made this script, and I have some thoughts and questions I'd like answered if possible.

First the script:

#region Script Settings
#http://tempuri.org/ScriptSettings.xsd">
#
# powershell.exe
#
# %TEMP%
#
# true
# false
# false
# true
# true
# false
# true
#
# 1.0.0.1
#
#
# false
# false
# false
# false
# false
#

#

#endregion

# ========================================================
#
#     Script Information
#
#    Title:                    FWLogUsers.ps1
#    Author:                    Jacob Saaby Nielsen, jsy@systematic.dk
#    Originally created:    05-11-2007 - 11:59:54
#    Original path:            C:\Documents and Settings\jsy\My Documents\AdminScriptEditor\FWLogUsers.ps1
#    Description:            Get usernames from logfiles, and their last logon time
# Syslog Utility:        KiwiSysLog
# Received Devices:     Cisco PIX 505, 515 and VPN 3000 Concentrator, Cisco
3550 Switch
#    
# ========================================================
function isUserStr ([string]$UserString)
{
    # Look for one of two patterns:
    #
    # Pattern 1: User [domain\user] Group [group] connected
    # Pattern 2: User [user] Group [group] connected
    #
    # RegEx for No. 1: User \x5B\w*\x5C\w*\x5D Group \x5B\w*\x5D connected
    # RegEx for No. 2: User \x5B\w*\x5D Group \x5B\w*\x5D connected

    # Set the regular expression to Pattern 1
    $Regex = [regex]"User \x5B\w*\x5C\w*\x5D Group \x5B\w*\x5D connected"
    # If the result of the match is true (the pattern we look for exists in
the string
    # we pass to the function), simply return true and exit the function
    if ($Regex.Ismatch($UserString) -eq $true)
    {
        return $true
    }
    else
    {
        # Set the regular expression to Pattern 2. The function only come this
far, if
        # the previous regex match returned false
        $Regex = [regex]"User \x5B\w*\x5D Group \x5B\w*\x5D connected"

        # If Pattern 2 is false, then return false, if not, return true
        if ($Regex.Ismatch($UserString) -eq $false)
            {
                return $false
            }
        else
            {
                return $true
            }
    }
}
function getTime ([string]$UserTime)
{
    # Look for time in a string
    # Regular Expression: \d{2}[:]\d{2}[:]\d{2}
    # Two digits, colon, two digits, colon, two digits

    # Set the $Regex variable to the actual regular expression you want to match
    $Regex = [regex]"\d{2}[:]\d{2}[:]\d{2}"

    # Set the $Match variable to the result of matching the string you pass,
    # with the regular expression
    $Match = $Regex.Match($userTime)

    # Return the value of the match
    return $Match.Value
}
function getDate ([string]$UserDate)
{
    # Look for a date in a string
    # Regular Expression: \d{4}[-]\d{2}[-]\d{2}
    # Four digits, dash, two digits, dash, two digits

    # Set the $Regex variable to the actual regular expression you want to match
    $Regex = [regex]"\d{4}[-]\d{2}[-]\d{2}"

    # Set the $Match variable to the result of matching the string you pass,
    # with the regular expression
    $Match = $Regex.Match($UserDate)

    # Return the value of the match
    return $Match.Value
}
function getUser([string]$UserUser)
{
    # Look for a username in a string

    # Set the $Regex variable to the actual regular expression you want to match
    # User [group\user]    
    $Regex = [regex] "User \x5B\w*\x5C\w*\x5D"    

    # If it matches the regular expression...
    if ($Regex.Ismatch($UserUser) -eq $true)
    {
        # Set the $Match variable to the result of matching the string you pass,
        # with the regular expression
        $Match = $Regex.Match($UserUser)    

        # Assign the value of the matched regular expression to the variable $UserName
        [string]$UserName = $Match.Value

        # Find out where the starting position, when copying the username, is
        [int]$StartPos = $UserName.IndexOf("\") + 1

        # Find out where the ending position, when copying the username, is
        [int]$EndPos = $UserName.IndexOf("]")

        # Find out how many characters to copy
        [int]$CopyLength = ($EndPos - $StartPos)

        # Return a substring that goes from the starting position, and X numbers of
        # characters forward
        return $UserName.Substring($StartPos, $CopyLength)
    }
    else
    {
        # This section not commented, as it is the same as the above statement
        # User [user]
        $Regex = [regex]"User \x5B\w*\x5D"

$Match = $Regex.Match($UserUser)

[string]$UserName = $Match.Value

        [int]$StartPos = $UserName.IndexOf("[") + 1
        [int]$EndPos = $UserName.IndexOf("]")
        [int]$CopyLength = ($EndPos - $StartPos)

        return $UserName.Substring($StartPos, $CopyLength)
    }
}

# Which directory I want the resulting text file in [string]$FWUsersLogResultsDir = "c:\"

# The name of the logfile will be e.g. 04122007_1122.csv [string]$FWUsersLogResultsFile = (Get-Date -format ddMMyyyy) + "_" + ( Get-Date -format HHmm ) + ".csv"

# Which folder will I be recursing through
[string]$FWUsersLogDir = "\\server\datedlogs"

# Initialization of hashtable
$userArray = @{ }
# For each logfile in the logdirectory
foreach ($logfile in Get-ChildItem $FWUsersLogDir -recurse)
{
    # That isn't a folder...
    if (!$logfile.PSIsContainer)
    {
        # For each line in the logfile
        foreach ($logline in Get-Content $logfile.Fullname)
        {
            # If the line contains user logon information
            if ((isUserStr $logline) -eq $true)
            {
                # Populate the hashtable
                $userArray[( getUser $logline )] =
                @{ logTime = getTime($logline);
                logDate = getDate($logline);
                logFile = $logfile.Fullname }
            }
        }
    }
}
# Export data to a comma separated file
foreach ($userName in $userArray.Keys | Sort-Object)
{
    Add-Content -path($FWUsersLogResultsDir + $FWUsersLogResultsFile) -value($userName
+ "," + $userArray[$userName].logTime + "," + $userArray[$userName].logDate

+ "," + $userArray[$userName].logFile)
}


-----------------------

And the afterthoughts:

running my logscript has made me ponder a few things, and I have some questions
that perhaps
you scripting pros especially could help me out with.

One of them being that maaaby, I should've done the appendfile thingy after each logfile hehe.

But you know, that's why I did the script, to learn from it. So I've thought
about it for some days,
and have some questions:

1. Is there a way to increase performance by reading blocks of data from
the logfiles, instead of
line by line ?

2. If I was to add a progress indicator, wouldn't that need to know how many
files in total it's looking
through, to be able to scale the indicator accordingly ? I'm thinking about
doing a progress indicator
for:

- Progress in relation to the current logfile
- In relation to the current folder
- In relation to the total number of files

3. I'm considering also displaying a counter for how many results it's found.
Is there a way to statically
define a variable that's shown on screen, and when the value is updated,
it's updated on screen, but
not in a new screen position ? (I know Rexx had this in OS/2)

4. Also considering displaying some metrics. How much data, how many files,
how many lines exactly,
etc. Any obvious cmdlets or methods to use here, other then getting filesizes,
counting lines in a log-
file, etc. ?

Hope you can help me out with those questions. It's funny how when you've
made a script like that, and
you're running it (especially on such amounts of data), the obvious lacks
tend to get in your face pretty
quick ;)

But it's all good, I'm hoping to learn from it, and I'm hoping you can help me out a bit, learning :)


JacobSaabyUser is Offline
New Member
New Member
Posts:17
Avatar

--
11 Dec 2007 09:53 AM  
I optimized it. Performs 56 times better than the previously posted one. Also added a progress bar, and results get written to disk, every time the script is done with a logfile.

#region Script Settings
#
#
# powershell.exe
#
# %TEMP%
#
# true
# false
# false
# true
# true
# false
# true
#
# 1.0.0.1
#
#
# false
# false
# false
# false
# false
#

#

#endregion

# ================================================================================================================
#
# Script Information
#
# Title: FWLogUsers.ps1
# Author: Jacob Saaby Nielsen, jsy@systematic.dk
# Originally created: 05-11-2007 - 11:59:54
# Original path: C:\Documents and Settings\jsy\My Documents\AdminScriptEditor\FWLogUsers.ps1
# Description: Get usernames from logfiles, and their last logon time
# Syslog Utility: KiwiSysLog
# Received Devices: Cisco PIX 505, 515 and VPN 3000 Concentrator, Cisco 3550 Switch
#
# Version 2: Added -readcount 1000 to get-content. This yielded a 5600% performance improvement
# (No kidding - 9m22s without, 10 seconds flat with).
# Changed add-content so content gets added after each logfile, instead of when all
# logfiles have been processed. One processed logfile equals one addition of data to
# the resulting logfile.
# Added progress indicator for Total Number Of Files.
#
#
# ================================================================================================================

function isUserStr ([string]$UserString)
{
# Look for one of two patterns:
#
# Pattern 1: User [domain\user] Group [group] connected
# Pattern 2: User [user] Group [group] connected
#
# RegEx for No. 1: User \x5B\w*\x5C\w*\x5D Group \x5B\w*\x5D connected
# RegEx for No. 2: User \x5B\w*\x5D Group \x5B\w*\x5D connected

# Set the regular expression to Pattern 1
$Regex = [regex]"User \x5B\w*\x5C\w*\x5D Group \x5B\w*\x5D connected"

# If the result of the match is true (the pattern we look for exists in the string
# we pass to the function), simply return true and exit the function
if ($Regex.Ismatch($UserString) -eq $true)
{
return $true
}
else
{
# Set the regular expression to Pattern 2. The function only come this far, if
# the previous regex match returned false
$Regex = [regex]"User \x5B\w*\x5D Group \x5B\w*\x5D connected"

# If Pattern 2 is false, then return false, if not, return true
if ($Regex.Ismatch($UserString) -eq $false)
{
return $false
}
else
{
return $true
}
}
}

function getTime ([string]$UserTime)
{
# Look for time in a string
# Regular Expression: \d{2}[:]\d{2}[:]\d{2}
# Two digits, colon, two digits, colon, two digits

# Set the $Regex variable to the actual regular expression you want to match
$Regex = [regex]"\d{2}[:]\d{2}[:]\d{2}"

# Set the $Match variable to the result of matching the string you pass,
# with the regular expression
$Match = $Regex.Match($userTime)

# Return the value of the match
return $Match.Value
}

function getDate ([string]$UserDate)
{
# Look for a date in a string
# Regular Expression: \d{4}[-]\d{2}[-]\d{2}
# Four digits, dash, two digits, dash, two digits

# Set the $Regex variable to the actual regular expression you want to match
$Regex = [regex]"\d{4}[-]\d{2}[-]\d{2}"

# Set the $Match variable to the result of matching the string you pass,
# with the regular expression
$Match = $Regex.Match($UserDate)

# Return the value of the match
return $Match.Value
}

function getUser([string]$UserUser)
{
# Look for a username in a string

# Set the $Regex variable to the actual regular expression you want to match
# User [group\user]
$Regex = [regex] "User \x5B\w*\x5C\w*\x5D"

# If it matches the regular expression...
if ($Regex.Ismatch($UserUser) -eq $true)
{
# Set the $Match variable to the result of matching the string you pass,
# with the regular expression
$Match = $Regex.Match($UserUser)

# Assign the value of the matched regular expression to the variable $UserName
[string]$UserName = $Match.Value

# Find out where the starting position, when copying the username, is
[int]$StartPos = $UserName.IndexOf("\") + 1

# Find out where the ending position, when copying the username, is
[int]$EndPos = $UserName.IndexOf("]")

# Find out how many characters to copy
[int]$CopyLength = ($EndPos - $StartPos)

# Return a substring that goes from the starting position, and X numbers of
# characters forward
return $UserName.Substring($StartPos, $CopyLength)
}
else
{
# This section not commented, as it is the same as the above statement
# User [user]
$Regex = [regex]"User \x5B\w*\x5D"

$Match = $Regex.Match($UserUser)

[string]$UserName = $Match.Value

[int]$StartPos = $UserName.IndexOf("[") + 1
[int]$EndPos = $UserName.IndexOf("]")
[int]$CopyLength = ($EndPos - $StartPos)

return $UserName.Substring($StartPos, $CopyLength)
}
}

# Which directory I want the resulting text file in
[string]$FWUsersLogResultsDir = "c:\powershell"

# The name of the logfile will be e.g. 04122007_1122.csv
[string]$FWUsersLogResultsFile = (Get-Date -format ddMMyyyy) + "_" + ( Get-Date -format HHmm ) + ".csv"

# Which folder will I be recursing through
[string]$FWUsersLogDir = "C:\Program Files\Syslogd\Dated logs"

# Initialization of hashtable
$userArray = @{ }

CLS
Write-Host "Counting how many logfiles are available..."

$TotalNumberOfFiles = (Get-ChildItem $FWUsersLogDir -recurse | where { !$_.psiscontainer }).length
$FileCounter = 0

CLS

# For each logfile in the logdirectory
foreach ($logfile in Get-ChildItem $FWUsersLogDir -recurse)
{
# That isn't a folder...
if (!$logfile.PSIsContainer)
{
$status = "Processing file {0} of {1}: {2}" -f $FileCounter, $TotalNumberOfFiles, $logFile.Name
Write-Progress $status -PercentComplete ((100 / $TotalNumberOfFiles) * $FileCounter ) -Activity "Processing logfiles" -ID 1

# For each line in the logfile
foreach ($logline in Get-Content $logfile.Fullname -readcount 1000)
{
# If the line contains user logon information
if ((isUserStr $logline) -eq $true)
{
# Populate the hashtable
$userArray[( getUser $logline )] = @{ logTime = getTime($logline); logDate = getDate($logline); logFile = $logfile.Fullname }
}
}

$FileCounter++
}

# Export data to a comma separated file
foreach ($userName in $userArray.Keys | Sort-Object)
{
Add-Content -path($FWUsersLogResultsDir + $FWUsersLogResultsFile) -value($userName + "," + $userArray[$userName].logTime + "," + $userArray[$userName].logDate + "," + $userArray[$userName].logFile) -ErrorAction continue
}

# Re-initialize the array
$userArray = @{ }
}
bsonposhUser is Offline
Basic Member
Basic Member
Posts:392
Avatar

--
11 Dec 2007 02:19 PM  
Processing files would be faster using System.IO.StreamReader

Here is an example of use.
http://www.codeplex.com/PsObject/WorkItem/View.aspx?WorkItemId=8520
Brandon Shell
----------------
Microsoft Powershell MVP
https://mvp.support.microsoft.com/profile/Brandon
Blog: http://www.bsonposh.com
You are not authorized to post a reply.

Active Forums 4.1
right
   
footer Sponsored by Quest Software • SAPIEN Technologies • ShellTools, LLC • Microsoft Windows Server 2008 footer
footer