diff --git a/Dockerfile b/Dockerfile index 9a42e3f..23839b4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ ARG EXT_DIR="$ULS_DIR/ext" ARG ETP_CLI_VERSION="0.3.7" ARG EAA_CLI_VERSION="0.4.6" -ARG MFA_CLI_VERSION="0.0.6" +ARG MFA_CLI_VERSION="0.0.7" # ENV VARS ENV ULS_DIR=$ULS_DIR @@ -64,6 +64,7 @@ RUN git clone --depth 1 -b "${MFA_CLI_VERSION}" --single-branch http pip3 install -r ${EXT_DIR}/cli-mfa/requirements.txt # ENTRYPOINTS / CMD +VOLUME ["${ULS_DIR}/var"] ENTRYPOINT ["/usr/local/bin/python3","-u","bin/uls.py"] #CMD ["--help"] # EOF diff --git a/README.md b/README.md index eaec2cb..6844ceb 100644 --- a/README.md +++ b/README.md @@ -43,8 +43,7 @@ It can be run directly as Python code, as a provided Docker container or through - [DNS](docs/LOG_OVERVIEW.md#dns) - [PROXY](docs/LOG_OVERVIEW.md#proxy) - [Akamai Phish-proof Multi Factor Authenticator (AKAMAI-MFA)](https://www.akamai.com/us/en/products/security/akamai-mfa.jsp) - - [AUTH](docs/LOG_OVERVIEW.md#authentication-logs-auth) - - [POLICY](docs/LOG_OVERVIEW.md#policy-logs-policy) + - [EVENT](docs/LOG_OVERVIEW.md#authentication-logs-auth) - Supported data outputs diff --git a/bin/config/global_config.py b/bin/config/global_config.py index 2994ac2..ef2f3be 100644 --- a/bin/config/global_config.py +++ b/bin/config/global_config.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Common global variables / constants -__version__ = "1.3.2" +__version__ = "1.3.3" __tool_name_long__ = "Akamai Unified Log Streamer" __tool_name_short__ = "ULS" @@ -29,7 +29,7 @@ # MFA bin_mfa_cli = "ext/cli-mfa/bin/akamai-mfa" # Path to the MFA CLI Executable -mfa_cli_feeds = ['POLICY', 'AUTH'] # Available MFA CLI feeds +mfa_cli_feeds = ['EVENT'] # Available MFA CLI feeds # INPUT Choices input_choices = ['EAA', 'ETP', 'MFA'] # Available input types diff --git a/bin/modules/UlsArgsParser.py b/bin/modules/UlsArgsParser.py index 0b044e2..bebd700 100644 --- a/bin/modules/UlsArgsParser.py +++ b/bin/modules/UlsArgsParser.py @@ -1,4 +1,4 @@ -# Copyright 2021 Akamai Technologies, Inc. All Rights Reserved +# Copyright 2022 Akamai Technologies, Inc. All Rights Reserved # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/bin/modules/UlsInputCli.py b/bin/modules/UlsInputCli.py index 1a8249d..6a6c0e2 100644 --- a/bin/modules/UlsInputCli.py +++ b/bin/modules/UlsInputCli.py @@ -1,4 +1,4 @@ -# Copyright 2021 Akamai Technologies, Inc. All Rights Reserved +# Copyright 2022 Akamai Technologies, Inc. All Rights Reserved # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -226,7 +226,7 @@ def proc_create(self): self.credentials_file_section, uls_config.edgerc_mfa) my_feed = self._feed_selector(self.feed, product_feeds) - cli_command = [self.bin_python, '-u', product_path, 'event', my_feed.lower(), '-f'] + cli_command = [self.bin_python, '-u', product_path, 'event', '-f'] cli_command[3:3] = self._uls_useragent(self.product, my_feed) cli_command[3:3] = edgegrid_auth cli_command[3:3] = self._prep_proxy(self.inproxy) diff --git a/bin/modules/UlsMonitoring.py b/bin/modules/UlsMonitoring.py index 893b31a..66c201b 100644 --- a/bin/modules/UlsMonitoring.py +++ b/bin/modules/UlsMonitoring.py @@ -1,4 +1,4 @@ -# Copyright 2021 Akamai Technologies, Inc. All Rights Reserved +# Copyright 2022 Akamai Technologies, Inc. All Rights Reserved # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/bin/modules/UlsOutput.py b/bin/modules/UlsOutput.py index 242ec77..251c77a 100644 --- a/bin/modules/UlsOutput.py +++ b/bin/modules/UlsOutput.py @@ -1,4 +1,4 @@ -# Copyright 2021 Akamai Technologies, Inc. All Rights Reserved +# Copyright 2022 Akamai Technologies, Inc. All Rights Reserved # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/bin/modules/UlsTools.py b/bin/modules/UlsTools.py index 36816a6..31010d3 100644 --- a/bin/modules/UlsTools.py +++ b/bin/modules/UlsTools.py @@ -1,4 +1,4 @@ -# Copyright 2021 Akamai Technologies, Inc. All Rights Reserved +# Copyright 2022 Akamai Technologies, Inc. All Rights Reserved # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/bin/modules/UlsTransformation.py b/bin/modules/UlsTransformation.py index 469871c..46dc3c2 100644 --- a/bin/modules/UlsTransformation.py +++ b/bin/modules/UlsTransformation.py @@ -1,4 +1,4 @@ -# Copyright 2021 Akamai Technologies, Inc. All Rights Reserved +# Copyright 2022 Akamai Technologies, Inc. All Rights Reserved # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/bin/modules/aka_log.py b/bin/modules/aka_log.py index 397e174..dedf1fe 100644 --- a/bin/modules/aka_log.py +++ b/bin/modules/aka_log.py @@ -1,4 +1,4 @@ -# Copyright 2021 Akamai Technologies, Inc. All Rights Reserved +# Copyright 2022 Akamai Technologies, Inc. All Rights Reserved # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -27,3 +27,4 @@ def init(loglevel='WARNING', loggername=None): return log # EOF + diff --git a/docs/ADDITIONAL_FEATURES.md b/docs/ADDITIONAL_FEATURES.md index a5c9be4..ae76e40 100644 --- a/docs/ADDITIONAL_FEATURES.md +++ b/docs/ADDITIONAL_FEATURES.md @@ -55,7 +55,7 @@ Please see the dedicated "[Transformations docs](TRANSFORMATIONS.md)" available. # AUTORESUME / RESUME This feature was introduced in ULS 1.3.0. -Different circumstances (network isse, server maintainence, ...) could lead to an interruption of the log stream or ULS itself. +Different circumstances (network issues, server maintainence, ...) could lead to an interruption of the log stream or ULS itself. As this could cause a gap in the continuous log delivery, ULS now offers the option to enable automated resume upon the last recorded checkpoint. AUTORESUME will create a checkpoint every 1000 lines of log (configureable) to prevent too many FS operations. diff --git a/docs/ARGUMENTS_ENV_VARS.md b/docs/ARGUMENTS_ENV_VARS.md index 2f62147..59013cc 100644 --- a/docs/ARGUMENTS_ENV_VARS.md +++ b/docs/ARGUMENTS_ENV_VARS.md @@ -12,17 +12,17 @@ The following tables list all available command line parameters and their corres ## INPUT -| Parameter | Env - Var | Options | Default | Description | -|---------------------------|-----------------|-------------------------------------------------------------------------------------------------------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| -i
--input | ULS_INPUT | 'EAA', 'ETP', 'MFA' | None | Specify the desired INPUT source | -| --feed | ULS_FEED | EAA: 'ACCESS', 'ADMIN', 'CONHEALTH'
ETP: 'THREAT', 'AUP', 'DNS', 'PROXY'
MFA: 'AUTH','POLICY' | None | Specify the desired INPUT feed | -| --format | ULS_FORMAT | 'JSON', 'TEXT' | JSON | Specify the desired INPUT (=OUTPUT) format | -| --inproxy
--inputproxy | ULS_INPUT_PROXY | HOST:PORT | None | Adjust proxy usage for INPUT data collection (cli)
If this parameter does not work as expected, [please read more about it here](./FAQ.md#--inputproxy-proxy-does-not-work-as-expected) | -| --rawcmd | ULS_RAWCMD | \ | None | USE with caution /!\
This is meant only to be used when told by AKAMAI [Click here for more information](ADDITIONAL_FEATURES.md#rawcmd---rawcmd-feature) | -| --edgerc | ULS_EDGERC | /path/to/your/.edgerc | '~/.edgerc' | Specify the location of the .edgerc EDGE GRID AUTH file | -| --section | ULS_SECTION | edgerc_config_section | 'default' | Specify the desired section within the .edgerc file | -| --starttime | ULS_STARTTIME | EPOCH timestamp | `cli_default` | Specify an EPOCH timestamp from where to start the log collection. | -| --endtime | ULS_ENDTIME | EPOCH timestamp | None | Specify an EPOCH timestamp up until where to fetch logs. ULS will exit after reaching this point.
ULS will not continue reading logs on CLI errors !!! | +| Parameter | Env - Var | Options | Default | Description | +|---------------------------|-----------------|------------------------------------------------------------------------------------------------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| -i
--input | ULS_INPUT | 'EAA', 'ETP', 'MFA' | None | Specify the desired INPUT source | +| --feed | ULS_FEED | EAA: 'ACCESS', 'ADMIN', 'CONHEALTH'
ETP: 'THREAT', 'AUP', 'DNS', 'PROXY'
MFA: 'EVENT' | None | Specify the desired INPUT feed | +| --format | ULS_FORMAT | 'JSON', 'TEXT' | JSON | Specify the desired INPUT (=OUTPUT) format | +| --inproxy
--inputproxy | ULS_INPUT_PROXY | HOST:PORT | None | Adjust proxy usage for INPUT data collection (cli)
If this parameter does not work as expected, [please read more about it here](./FAQ.md#--inputproxy-proxy-does-not-work-as-expected) | +| --rawcmd | ULS_RAWCMD | \ | None | USE with caution /!\
This is meant only to be used when told by AKAMAI [Click here for more information](ADDITIONAL_FEATURES.md#rawcmd---rawcmd-feature) | +| --edgerc | ULS_EDGERC | /path/to/your/.edgerc | '~/.edgerc' | Specify the location of the .edgerc EDGE GRID AUTH file | +| --section | ULS_SECTION | edgerc_config_section | 'default' | Specify the desired section within the .edgerc file | +| --starttime | ULS_STARTTIME | EPOCH timestamp | `cli_default` | Specify an EPOCH timestamp from where to start the log collection. | +| --endtime | ULS_ENDTIME | EPOCH timestamp | None | Specify an EPOCH timestamp up until where to fetch logs. ULS will exit after reaching this point.
ULS will not continue reading logs on CLI errors !!! | ## OUTPUT diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 11b00b9..5bd006b 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,4 +1,14 @@ # Version History +## v1.3.3 +||| +|---|---| +|Date|2022-02-28 +|Kind| Bugfix release +|Author|mschiess@akamai.com +- **Bugfix** + - Adopted to new MFA CLI Version (only single feed "EVENT" available anymore) + - Amended new dates to the file headers + - Added volume to dockerfile as data storage for "autoresume" ## v1.3.2 ||| diff --git a/docs/COMMAND_LINE_USAGE.md b/docs/COMMAND_LINE_USAGE.md index a95f367..48d0f6b 100644 --- a/docs/COMMAND_LINE_USAGE.md +++ b/docs/COMMAND_LINE_USAGE.md @@ -91,7 +91,7 @@ All log output will be directed to STDOUT by default. - MFA AUTH LOG ==> HTTP LISTENER (SPLUNK) disabled TLS verification ```bash - python3 bin/uls.py --input=MFA --feed auth --output HTTP --httpformat '{"event": %s}' --httpauthheader '{"Authorization": "Splunk xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"}' --httpurl "https://127.0.0.1:9091/services/collector/event" --httpinsecure + python3 bin/uls.py --input=MFA --feed event --output HTTP --httpformat '{"event": %s}' --httpauthheader '{"Authorization": "Splunk xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"}' --httpurl "https://127.0.0.1:9091/services/collector/event" --httpinsecure ``` - Logging to a file and sending process to the background diff --git a/docs/DOCKER-COMPOSE_USAGE.md b/docs/DOCKER-COMPOSE_USAGE.md index e015a8c..2608d60 100644 --- a/docs/DOCKER-COMPOSE_USAGE.md +++ b/docs/DOCKER-COMPOSE_USAGE.md @@ -67,4 +67,4 @@ docker compose up -d - [docker-compose.yml](examples/docker-compose/complex/docker-compose.yml) - [etp-threat.env](examples/docker-compose/complex/etp-threat.env) - [eaa-admin.env](examples/docker-compose/complex/eaa-access.env) - - [eaa-access.env](examples/docker-compose/complex/eaa-access.env) \ No newline at end of file + - [mfa.env](examples/docker-compose/complex/mfa.env) \ No newline at end of file diff --git a/docs/TRANSFORMATIONS.md b/docs/TRANSFORMATIONS.md index 3357f47..5e9d1a1 100644 --- a/docs/TRANSFORMATIONS.md +++ b/docs/TRANSFORMATIONS.md @@ -49,4 +49,9 @@ bin/uls.py --input eaa --feed access --section akamaidemo --output raw --transfo # EAA ACCESSS logs transformed to output only country, state and city in LIST format (sent to RAW output) bin/uls.py --input eaa --feed access --section akamaidemo --output raw --transformation jmespath --transformationpattern '[geo_country, geo_state, geo_city]' + +# EAA ACCESS logs transformed to a "NON-JSON" syslog'ish format, joining a static text with dynamic json values (dropping the keys) +pattern='join(``, [`"<14>uls-hostname {"`, query.time, query.domain, `"}"`])' +bin/uls.py --section akamaidemo --input etp --feed dns --output raw --transformation jmespath --transformationpattern "$pattern" ``` + diff --git a/docs/examples/docker-compose/complex/README.md b/docs/examples/docker-compose/complex/README.md index c7af2c5..328449b 100644 --- a/docs/examples/docker-compose/complex/README.md +++ b/docs/examples/docker-compose/complex/README.md @@ -17,7 +17,7 @@ All docker related controls can be found in [docker-compose.yml](docker-compose. ## MFA-AUTH |IN|OUT|FILE| |---|---|---| -|MFA-AUTH|UDP|[mfa-auth.env](mfa-auth.env)| +|MFA-AUTH|UDP|[mfa-auth.env](mfa.env)| diff --git a/docs/examples/docker-compose/complex/docker-compose.yml b/docs/examples/docker-compose/complex/docker-compose.yml index ca5d012..050050c 100644 --- a/docs/examples/docker-compose/complex/docker-compose.yml +++ b/docs/examples/docker-compose/complex/docker-compose.yml @@ -18,10 +18,10 @@ services: source: /path/to/your/.edgerc target: /opt/akamai-uls/.edgerc read_only: true - eaa-admin: + mfa: image: akamai/uls:latest restart: always - env_file: mfa-auth.env + env_file: mfa.env volumes: - type: bind source: /path/to/your/.edgerc diff --git a/docs/examples/docker-compose/complex/mfa-auth.env b/docs/examples/docker-compose/complex/mfa.env similarity index 94% rename from docs/examples/docker-compose/complex/mfa-auth.env rename to docs/examples/docker-compose/complex/mfa.env index dec30aa..12a9a3b 100644 --- a/docs/examples/docker-compose/complex/mfa-auth.env +++ b/docs/examples/docker-compose/complex/mfa.env @@ -5,7 +5,7 @@ ULS_LOGLEVEL=INFO # INPUT CONFIGURATION ULS_INPUT=MFA - ULS_FEED=AUTH + ULS_FEED=EVENT ULS_FORMAT=JSON ULS_EDGERC=/opt/akamai-uls/.edgerc diff --git a/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml b/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml index 8724d9d..e8ee9f2 100644 --- a/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml +++ b/docs/examples/kubernetes/helm/akamai-uls/Chart.yaml @@ -3,5 +3,5 @@ name: akamai-uls description: Akamai Universal Log Streamer Helm installation type: application -version: 0.1.0 -appVersion: "1.3.2" +version: 0.1.1 +appVersion: "1.3.3" diff --git a/test/positive_test.bats b/test/positive_test.bats index ac7258b..e110f11 100644 --- a/test/positive_test.bats +++ b/test/positive_test.bats @@ -81,18 +81,8 @@ load 'test/bats/bats-assert/load.bash' } ## MFA -@test "MFA - AUTH" { - skip "MFA API currently broken" - run timeout --preserve-status $uls_test_timeout $uls_bin --input mfa --feed auth --output raw --edgerc $uls_edgerc --section $uls_section - assert_output "" - #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" - #[ "$status" -eq 124 ] #return value from timeout without --preserve status - [ "$status" -eq 100 ] #return value from uls when interrupted --> with --preserve status on timeout -} - -@test "MFA - POLICY" { - skip "MFA API currently broken" - run timeout --preserve-status $uls_test_timeout $uls_bin --input mfa --feed policy --output raw --edgerc $uls_edgerc --section $uls_section +@test "MFA - EVENT" { + run timeout --preserve-status $uls_test_timeout $uls_bin --input mfa --feed event --output raw --edgerc $uls_edgerc --section $uls_section assert_output "" #assert_output --partial "The specified directory tmp does not exist or privileges are missing - exiting" #[ "$status" -eq 124 ] #return value from timeout without --preserve status diff --git a/test/test.sh b/test/test.sh index 6e47d24..f8c3a64 100644 --- a/test/test.sh +++ b/test/test.sh @@ -17,7 +17,7 @@ function do_test() { } function do_tag() { - my_date=$(date +%Y%M%d-%H%M%S-%Z) + my_date=$(date +%Y%m%d-%H%M%S-%Z) git tag "BATS-TEST-SUCCESSFUL__$my_date" }