"""
Foreman and Candlepin logs
==========================
Module for parsing the log files in foreman-debug archive
.. note::
Please refer to its super-class :class:`insights.core.LogFileOutput` for
usage information.
Parsers provided by this module:
CandlepinErrorLog - file ``sos_commands/foreman/foreman-debug/var/log/candlepin/error.log``
-------------------------------------------------------------------------------------------
CandlepinLog - file ``/var/log/candlepin/candlepin.log``
--------------------------------------------------------
ProductionLog - file ``/var/log/foreman/production.log``
--------------------------------------------------------
ProxyLog - file ``/var/log/foreman-proxy/proxy.log``
----------------------------------------------------
SatelliteLog - file ``/var/log/foreman-installer/satellite.log``
----------------------------------------------------------------
ForemanSSLAccessLog - file ``/var/log/httpd/foreman-ssl_access_ssl.log``
------------------------------------------------------------------------
ForemanSSLErrorLog - file ``/var/log/httpd/foreman-ssl_error_ssl.log``
----------------------------------------------------------------------
"""
from datetime import datetime
from .. import LogFileOutput, parser
from insights.specs import Specs
[docs]
@parser(Specs.foreman_proxy_log)
class ProxyLog(LogFileOutput):
"""Class for parsing ``foreman-proxy/proxy.log`` file."""
time_format = {
'standard': '%d/%b/%Y:%H:%M:%S', # 31/May/2016:09:57:34
'error': '%Y-%m-%dT%H:%M:%S.%f' # 2016-05-31T09:57:35.884636
}
[docs]
@parser(Specs.foreman_satellite_log)
class SatelliteLog(LogFileOutput):
"""Class for parsing ``foreman-installer/satellite.log`` file."""
pass
[docs]
@parser(Specs.foreman_production_log)
class ProductionLog(LogFileOutput):
"""Class for parsing ``foreman/production.log`` file.
Each line is parsed into a dictionary with the following keys:
* raw_message (str) - complete log line
* message (str) - the body of the log
* timestamp (datetime) - date and time of log as datetime object
"""
def _parse_line(self, line):
msg_info = {'raw_message': line}
line_split = line.split(None, 2)
if len(line_split) > 2:
try:
msg_info['timestamp'] = datetime.strptime(' '.join(line_split[:2]), self.time_format)
msg_info['message'] = line_split[2]
except ValueError:
pass
return msg_info
[docs]
@parser(Specs.candlepin_log)
class CandlepinLog(LogFileOutput):
"""Class for parsing ``candlepin/candlepin.log`` file.
Sample input::
2016-09-09 13:45:52,650 [req=bd5a4284-d280-4fc5-a3d5-fc976b7aa5cc, org=] INFO org.candlepin.common.filter.LoggingFilter - Request: verb=GET, uri=/candlepin/consumers/f7677b4b-c470-4626-86a4-2fdf2546af4b
2016-09-09 13:45:52,784 [req=bd5a4284-d280-4fc5-a3d5-fc976b7aa5cc, org=example_org] INFO org.candlepin.common.filter.LoggingFilter - Response: status=200, content-type="application/json", time=134
2016-09-09 13:45:52,947 [req=909ca4c5-f24e-4212-8f23-cc754d06ac57, org=] INFO org.candlepin.common.filter.LoggingFilter - Request: verb=GET, uri=/candlepin/consumers/f7677b4b-c470-4626-86a4-2fdf2546af4b/content_overrides
2016-09-09 13:45:52,976 [req=909ca4c5-f24e-4212-8f23-cc754d06ac57, org=] INFO org.candlepin.common.filter.LoggingFilter - Response: status=200, content-type="application/json", time=29
2016-09-09 13:45:53,072 [req=49becd26-5dfe-4d2f-8667-470519230d88, org=] INFO org.candlepin.common.filter.LoggingFilter - Request: verb=GET, uri=/candlepin/consumers/f7677b4b-c470-4626-86a4-2fdf2546af4b/release
2016-09-09 13:45:53,115 [req=49becd26-5dfe-4d2f-8667-470519230d88, org=example_org] INFO org.candlepin.common.filter.LoggingFilter - Response: status=200, content-type="application/json", time=43
Each line is parsed into a dictionary with the following keys:
* **raw_message(str)** - complete log line
* **message(str)** - the body of the log
* **timestamp(datetime)** - date and time of log as datetime object
Examples:
>>> cp_log_lines = cp_log.get('candlepin/consumers')
>>> len(cp_log_lines)
3
>>> cp_log_lines[0].get('raw_message')
'2016-09-09 13:45:52,650 [req=bd5a4284-d280-4fc5-a3d5-fc976b7aa5cc, org=] INFO org.candlepin.common.filter.LoggingFilter - Request: verb=GET, uri=/candlepin/consumers/f7677b4b-c470-4626-86a4-2fdf2546af4b'
>>> cp_log_lines[0].get('message')
'[req=bd5a4284-d280-4fc5-a3d5-fc976b7aa5cc, org=] INFO org.candlepin.common.filter.LoggingFilter - Request: verb=GET, uri=/candlepin/consumers/f7677b4b-c470-4626-86a4-2fdf2546af4b'
>>> cp_log_lines[0].get('timestamp')
datetime.datetime(2016, 9, 9, 13, 45, 52, 650000)
"""
time_format = '%Y-%m-%d %H:%M:%S,%f'
def _parse_line(self, line):
# line format from /var/lib/tomcat/webapps/candlepin/WEB-INF/classes/logback.xml
# <pattern>%d{ISO8601} [thread=%thread] [%X{requestType}=%X{requestUuid}, org=%X{org}, csid=%X{csid}] %-5p %c - %m%n</pattern>
# http://logback.qos.ch/manual/layouts.html
msg_info = {'raw_message': line}
line_split = line.split(None, 2)
if len(line_split) > 2:
try:
msg_info['timestamp'] = datetime.strptime(' '.join(line_split[:2]), self.time_format)
msg_info['message'] = line_split[2]
except ValueError:
pass
return msg_info
[docs]
@parser(Specs.candlepin_error_log)
class CandlepinErrorLog(LogFileOutput):
"""
Class for parsing ``candlepin/error.log`` file.
Sample log contents::
2016-09-07 13:56:49,001 [=, org=] WARN org.apache.qpid.transport.network.security.ssl.SSLUtil - Exception received while trying to verify hostname
2016-09-07 14:07:33,735 [=, org=] WARN org.apache.qpid.transport.network.security.ssl.SSLUtil - Exception received while trying to verify hostname
2016-09-07 14:09:55,173 [=, org=] WARN org.apache.qpid.transport.network.security.ssl.SSLUtil - Exception received while trying to verify hostname
2016-09-07 15:20:33,796 [=, org=] WARN org.apache.qpid.transport.network.security.ssl.SSLUtil - Exception received while trying to verify hostname
2016-09-07 15:27:34,367 [=, org=] WARN org.apache.qpid.transport.network.security.ssl.SSLUtil - Exception received while trying to verify hostname
2016-09-07 16:49:24,650 [=, org=] WARN org.apache.qpid.transport.network.security.ssl.SSLUtil - Exception received while trying to verify hostname
2016-09-07 18:07:53,688 [req=d9dc3cfd-abf7-485e-b1eb-e1e28e4b0f28, org=org_ray] ERROR org.candlepin.sync.Importer - Conflicts occurred during import that were not overridden:
2016-09-07 18:07:53,690 [req=d9dc3cfd-abf7-485e-b1eb-e1e28e4b0f28, org=org_ray] ERROR org.candlepin.sync.Importer - [DISTRIBUTOR_CONFLICT]
2016-09-07 18:07:53,711 [req=d9dc3cfd-abf7-485e-b1eb-e1e28e4b0f28, org=org_ray] ERROR org.candlepin.resource.OwnerResource - Recording import failure org.candlepin.sync.ImportConflictException: Owner has already imported from another subscription management application.
Examples:
>>> candlepin_log.get('req=d9dc3cfd-abf7-485e-b1eb-e1e28e4b0f28')[0]['raw_message']
'2016-09-07 18:07:53,688 [req=d9dc3cfd-abf7-485e-b1eb-e1e28e4b0f28, org=org_ray] ERROR org.candlepin.sync.Importer - Conflicts occurred during import that were not overridden:'
>>> from datetime import datetime
>>> list(candlepin_log.get_after(datetime(2016, 9, 7, 16, 0, 0)))[0]['raw_message']
'2016-09-07 16:49:24,650 [=, org=] WARN org.apache.qpid.transport.network.security.ssl.SSLUtil - Exception received while trying to verify hostname'
"""
pass
[docs]
@parser(Specs.foreman_ssl_access_ssl_log)
class ForemanSSLAccessLog(LogFileOutput):
"""Class for parsing ``/var/log/httpd/foreman-ssl_access_ssl.log`` file.
Sample log contents::
10.181.73.211 - rhcapkdc.example2.com [27/Mar/2017:13:34:52 -0400] "GET /rhsm/consumers/385e688f-43ad-41b2-9fc7-593942ddec78 HTTP/1.1" 200 10736 "-" "-"
10.181.73.211 - rhcapkdc.example2.com [27/Mar/2017:13:34:52 -0400] "GET /rhsm/status HTTP/1.1" 200 263 "-" "-"
10.185.73.33 - 8a31cd915917666001591d6fb44602a7 [27/Mar/2017:13:34:52 -0400] "GET /pulp/repos/Acme_Inc/Library/RHEL7_Sat_Cap sule_Servers/content/dist/rhel/server/7/7Server/x86_64/os/repodata/repomd.xml HTTP/1.1" 200 2018 "-" "urlgrabber/3.10 yum/3.4.3"
10.181.73.211 - rhcapkdc.example2.com [27/Mar/2017:13:34:52 -0400] "GET /rhsm/consumers/4f8a39d0-38b6-4663-8b7e-03368be4d3ab/owner HTTP/1.1" 200 5159 "-"
10.181.73.211 - rhcapkdc.example2.com [27/Mar/2017:13:34:52 -0400] "GET /rhsm/consumers/385e688f-43ad-41b2-9fc7-593942ddec78/compliance HTTP/1.1" 200 5527
10.181.73.211 - rhcapkdc.example2.com [27/Mar/2017:13:34:52 -0400] "GET /rhsm/consumers/4f8a39d0-38b6-4663-8b7e-03368be4d3ab HTTP/1.1" 200 10695 "-" "-"
Each line is parsed into a dictionary with the following keys:
* **raw_message(str)** - complete log line
* **host(str)** - remote host's IP or hostname
* **timestamp(datetime)** - date and time of http request, time zone is ignored
Examples:
>>> foreman_ssl_acess_log.get('consumers/385e688f-43ad-41b2-9fc7-593942ddec78')[0]['raw_message']
'10.181.73.211 - rhcapkdc.example2.com [27/Mar/2017:13:34:52 -0400] "GET /rhsm/consumers/385e688f-43ad-41b2-9fc7-593942ddec78 HTTP/1.1" 200 10736 "-" "-"'
>>> foreman_ssl_acess_log.get('consumers/385e688f-43ad-41b2-9fc7-593942ddec78')[0]['host']
'10.181.73.211'
>>> foreman_ssl_acess_log.get('consumers/385e688f-43ad-41b2-9fc7-593942ddec78')[0]['timestamp']
datetime.datetime(2017, 3, 27, 13, 34, 52)
"""
# Actual time format - '%d/%b/%Y:%H:%M:%S %z' but %z doesn't work for python 2
time_format = '%d/%b/%Y:%H:%M:%S'
def _parse_line(self, line):
msg_info = {'raw_message': line}
# Split enough to get datetime and host
line_split = line.split(None, 4)
if len(line_split) == 5:
try:
msg_info['timestamp'] = datetime.strptime(
line_split[3].strip('['), self.time_format
)
msg_info['host'] = line_split[0]
except ValueError:
pass
return msg_info
[docs]
@parser(Specs.foreman_ssl_error_ssl_log)
class ForemanSSLErrorLog(LogFileOutput):
"""
Class for parsing ``/var/log/httpd/foreman-ssl_error_ssl.log`` file.
.. note::
Please refer to its super-class :class:`insights.core.LogFileOutput` for examples.
Sample log contents::
[Mon Aug 09 10:01:22.548717 2021] [ssl:warn] [pid 5881] [client 10.72.44.126:47190] AH02227: Failed to set r->user to 'SSL_CLIENT_S_DN_CN'
[Mon Aug 09 11:02:23.229609 2021] [proxy_http:error] [pid 749] (20014)Internal error: [client 10.72.44.126:47920] AH01102: error reading status line from remote server yyy
[Mon Aug 09 11:17:52.204503 2021] [proxy_http:error] [pid 5854] (20014)Internal error: [client 10.72.44.126:48016] AH01102: error reading status line from remote server yyy
"""
time_format = '%a %b %d %H:%M:%S.%f %Y'