Python: Snippet/Experiment – Syslog Server mit globalen und Host Filtern

Der Code ist nicht fertig und war mal ein Labor Versuch. Es lassen sich globale und Host Filter setzen wo diese zutreffen werden die Logs in ein extra File geschrieben.

Config file:

#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Config definition


class CFG:

    def __init__(self):

        # Path for logfiles
        self.syslogpath = "/home/mthoma/_dev/syslog/log/"

        # Listner Port
        self.port = 3702

        # Listner address
        self.host = "0.0.0.0"

        # Global Filter
        self.global_filter = {
            "filter": [
                ".*FOOBAR.*",
                ".*COFFEE.*"
            ]
        }

        # Host Filter
        self.host_filter = {
            "10.201.11.33": {
                "filter": [
                    ".*MACFLAP.*",
                    ".*BUBU.*",
                ]
            },
        }

Syslog Server:

#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Load config class
from config import CFG

# Load common classes
import re
import logging
import SocketServer
import socket
import os

# Load configuration file
C = CFG()

formatter = logging.Formatter('%(message)s')

def setup_logger(name, log_file, level=logging.INFO):
    handler = logging.FileHandler(log_file)
    handler.setFormatter(formatter)
    
    logger = logging.getLogger(name)
    logger.setLevel(level)
    logger.addHandler(handler)
    
    return logger


class SyslogUDPHandler(SocketServer.BaseRequestHandler):

    def handle(self):
        data = bytes.decode(self.request[0].strip())
        sockets = self.request[1]

        ip = str(self.client_address[0])
        
    # Try to resolve reverse record via DNS
        try:
            name, alias, addresslist = socket.gethostbyaddr(ip)
        except:
            name = ip
        
    # Set path
        path = C.syslogpath+name+"/"
        
    # Create path if not exist
        try:
            os.stat(path)
        except:
            os.mkdir(path)
        
        logger = setup_logger('normal_log', path+"log")
        logger.info(str(data))
        
        logger_sp = setup_logger('special_log', path+"spec")
        
        if ip in C.host_filter:
            filters = options['filter'] + C.global_filter['filter']
            filter_join = "|".join(filters)
            
            if re.match(r"%s" % filter_join, str(data)):
                logger_sp.info(str(data))
        else:
            filters = C.global_filter['filter']
            filter_join = "|".join(filters)
            
            if re.match(r"%s" % filter_join, str(data)):
                logger_sp.info(str(data))
                
        
        
        print "%s : " % self.client_address[0], str(data)

        logging.info(str(data))



if __name__ == "__main__":

    try:
        server = SocketServer.UDPServer((C.host,C.port), SyslogUDPHandler)
        server.serve_forever(poll_interval=0.5)

    except (IOError, SystemExit):
        raise

    except KeyboardInterrupt:
        print "Crtl+C Pressed. Shutting down."

 

Python: Snippet Multiprocessing mit Ergebnis

Beispiel für Parallelisierung von Jobs mit Ergebnis welche als Liste zurückgegeben werden.

#!/usr/bin/env python
# -*- encoding: utf-8; py-indent-offset: 4 -*-

import os
from multiprocessing import Pool


def worker(job):
    x, y = job
    result = x ** y
    return os.getpid(), result
  
if __name__ == '__main__':
    jobs = [(1, 2), (3, 4), (5, 6), (11, 12), (13, 14), (15, 16), (21, 22), (23, 24), (25, 26)]
    
    result_buffer = []
  
    pool = Pool(processes=5)
    
    for job in jobs:
        result_buffer.append(pool.apply_async(worker, args=(job,)))
    
    pool.close()
    pool.join()
  
    results = [r.get() for r in result_buffer]

    print results
  
    for pid, result in results:
        print "working pid was: %s" % pid
        print "result is: %s" % result
        print "---"

Beispiel Ergebnis:

$python mp_with_result.py

[(7992, 1), (7992, 81), (7992, 15625), (7992, 3138428376721L), (7992, 3937376385699289L), (7992, 6568408355712890625L), (7992, 122694327386105632949003612841L), (7992, 480250763996501976790165756943041L), (7992, 2220446049250313080847263336181640625L)]
working pid was: 7992
result is: 1
---
working pid was: 7992
result is: 81
---
working pid was: 7992
result is: 15625
---
working pid was: 7992
result is: 3138428376721
---
working pid was: 7992
result is: 3937376385699289
---
working pid was: 7992
result is: 6568408355712890625
---
working pid was: 7992
result is: 122694327386105632949003612841
---
working pid was: 7992
result is: 480250763996501976790165756943041
---
working pid was: 7992
result is: 2220446049250313080847263336181640625
---

 

Python: Experiment/Snippet – Komprimieren und löschen von Logfiles nach X Tagen

Ein Ansatz für Logverzeichnisse im Format /log/<yyyy>/<mm>/<dd>/<div. logsfiles>

#!/usr/bin/env python

import gzip
import shutil
import os
import datetime
import time

#############################################
# Config
#############################################

# Path of Logfiles
# Structure is /opt/log/<YYYY>/<MM>/<DD>/
gpath='/opt/log/'

# hold logs for x days
hold_time=180



#############################################

def get_immediate_subdirectories(a_dir):
    return [name for name in os.listdir(a_dir) if os.path.isdir(os.path.join(a_dir, name))]

def delete_files(f):
    # delete file if older than hold time
    nowx = time.time()

    for file in os.listdir(f):    
        if os.stat(f+file).st_mtime < nowx - hold_time * 86400:
            f_path = f+file
            print "delete %s " % f_path
            os.remove(f_path)
            
    try:
        os.rmdir(f)
    except:
        pass
            
    
    
def compress_files(lpath):
    # Compress files
    print "Working on: " + lpath
    obj = os.listdir(lpath)
    for f in obj:
        if os.path.isfile(lpath+f) and ".gz" not in f:
            with open(lpath+f,'rb') as f_in:
                with gzip.open(lpath+f+".gz",'wb') as f_out:
                    shutil.copyfileobj(f_in, f_out)
                    os.remove(lpath+f)
            
#compress everything which ist older than now

now = datetime.datetime.now()
years = get_immediate_subdirectories(gpath)

for year in years:

    # delete empty directories
    if not os.listdir(gpath+year):
        os.rmdir(gpath+year)
    else:

        months = get_immediate_subdirectories(gpath+year)
    
        for month in months:

            # delete empty directories
            if not os.listdir(gpath+year+"/"+month):
                os.rmdir(gpath+year+"/"+month)

            else:
                days = get_immediate_subdirectories(gpath+year+"/"+month)
                        
                # Remove current day from compressing & cleaning
                if month == str(now.month) and year == str(now.year):

                    if len(str(now.day)):
                        now_day = "0%s" % now.day
                    else:
                        now_day = str(now.day)

                    days.remove(now_day)
        
                for day in days:
                    # delete empty directories
                    if not os.listdir(gpath+year+"/"+month+"/"+day+"/"):
                        os.rmdir(gpath+year+"/"+month+"/"+day+"/")
                    else:
                        # compress all files in folder
                        compress_files(gpath+year+"/"+month+"/"+day+"/")
                        
                        # delete old files
                        delete_files(gpath+year+"/"+month+"/"+day+"/")