From af259e3be93f266e18069b46f8eeac47c18ef23b Mon Sep 17 00:00:00 2001 From: "Gabriele A. Ron" Date: Wed, 7 Feb 2024 14:00:14 -0600 Subject: [PATCH 01/15] Bumped version --- .bumpversion.cfg | 2 +- CHANGELOG.md | 11 +++++++++++ src/forge/__init__.py | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index a5a17bb..e76a82f 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.2 +current_version = 1.0.3 commit = True tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6caa440..c5e64b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased] +## [1.0.3] - 2024-02-07 + +### Added +- **Create** + - Multi-AZ functionality + - Spot retries + - On-demand Failover + +### Changed +- **Create** - Configurable spot strategy +- **Documentation** - Updated with new changes ## [1.0.2] - 2022-10-27 diff --git a/src/forge/__init__.py b/src/forge/__init__.py index 2a8cc67..5158a7c 100755 --- a/src/forge/__init__.py +++ b/src/forge/__init__.py @@ -1,4 +1,4 @@ -__version__ = "1.0.2" +__version__ = "1.0.3" # Default values for forge's essential arguments DEFAULT_ARG_VALS = { From dcd91332aeafa95da64ada71aee9be830204d976 Mon Sep 17 00:00:00 2001 From: "Gabriele A. Ron" Date: Wed, 7 Feb 2024 14:03:19 -0600 Subject: [PATCH 02/15] Bumped version to 1.1.0 --- .bumpversion.cfg | 2 +- CHANGELOG.md | 2 +- src/forge/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index e76a82f..73dded6 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.3 +current_version = 1.1.0 commit = True tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5e64b4..04d6e65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased] -## [1.0.3] - 2024-02-07 +## [1.1.0] - 2024-02-07 ### Added - **Create** diff --git a/src/forge/__init__.py b/src/forge/__init__.py index 5158a7c..a96e573 100755 --- a/src/forge/__init__.py +++ b/src/forge/__init__.py @@ -1,4 +1,4 @@ -__version__ = "1.0.3" +__version__ = "1.1.0" # Default values for forge's essential arguments DEFAULT_ARG_VALS = { From 97c15ba3b28823eafb2fc44467bec872fa8a8b0d Mon Sep 17 00:00:00 2001 From: "Gabriele A. Ron" Date: Wed, 7 Feb 2024 15:06:21 -0600 Subject: [PATCH 03/15] Updated maintainers --- pyproject.toml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 981805f..b0440f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,14 @@ readme = "README.md" requires-python = ">=3.7" license = "Apache-2.0" authors = [ - {name = "Nikhil Patel", email = "npatel@cars.com"} + {name = "Nikhil Patel", email = "npatel@cars.com"}, + {name = "Gabriele Ron", email = "gron@cars.com"}, + {name = "Joao Moreira", email = "jmoreira@cars.com"} +] +maintainers = [ + {name = "Nikhil Patel", email = "npatel@cars.com"}, + {name = "Gabriele Ron", email = "gron@cars.com"}, + {name = "Joao Moreira", email = "jmoreira@cars.com"} ] keywords = [ "AWS", From e1f8653111c485d95f798fc75fa619b9da9343cb Mon Sep 17 00:00:00 2001 From: "Gabriele A. Ron" Date: Thu, 8 Feb 2024 09:40:50 -0600 Subject: [PATCH 04/15] 1.1.1 --- .bumpversion.cfg | 2 +- CHANGELOG.md | 5 +++++ src/forge/__init__.py | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 73dded6..32da4e3 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.1.0 +current_version = 1.1.1 commit = True tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04d6e65..c5a4a19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased] +## [1.1.1] - 2024-02-08 + +### Changed +- **Create** - Set default boto3 session at beginning of create to resolve region bug + ## [1.1.0] - 2024-02-07 ### Added diff --git a/src/forge/__init__.py b/src/forge/__init__.py index a96e573..1781885 100755 --- a/src/forge/__init__.py +++ b/src/forge/__init__.py @@ -1,4 +1,4 @@ -__version__ = "1.1.0" +__version__ = "1.1.1" # Default values for forge's essential arguments DEFAULT_ARG_VALS = { From 1ee83a9a0bc4752dd3423c4c4f0ae0b3692b36a0 Mon Sep 17 00:00:00 2001 From: "Gabriele A. Ron" Date: Fri, 9 Feb 2024 11:34:31 -0600 Subject: [PATCH 05/15] Fixed automatic multi-worker allocation bug --- src/forge/create.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/forge/create.py b/src/forge/create.py index 8649190..47ff425 100755 --- a/src/forge/create.py +++ b/src/forge/create.py @@ -703,11 +703,12 @@ def _check(x, i): return x[i] if x and x[i:i + 1] else None for task in task_list: + task_worker_count = worker_count if 'cluster-master' in task or 'single' in task: task_ram, task_cpu, total_ram, ram2cpu_ratio = calc_machine_ranges(ram=_check(ram, 0), cpu=_check(cpu, 0), ratio=_check(ratio, 0)) - worker_count = 1 + task_worker_count = 1 elif 'cluster-worker' in task: - task_ram, task_cpu, total_ram, ram2cpu_ratio = calc_machine_ranges(ram=_check(ram, 1), cpu=_check(cpu, 1), ratio=_check(ratio, 1), workers=worker_count) + task_ram, task_cpu, total_ram, ram2cpu_ratio = calc_machine_ranges(ram=_check(ram, 1), cpu=_check(cpu, 1), ratio=_check(ratio, 1), workers=task_worker_count) else: logger.error("'%s' does not seem to be a valid cluster or single job.", task) if destroy_flag: @@ -717,8 +718,8 @@ def _check(x, i): logger.debug('%s OVERRIDE DETAILS | RAM: %s out of %s | CPU: %s with ratio of %s', task, ram, total_ram, cpu, ram2cpu_ratio) instance_details[task] = { - 'total_capacity': worker_count or total_ram, - 'capacity_unit': 'units' if worker_count else 'memory-mib', + 'total_capacity': task_worker_count or total_ram, + 'capacity_unit': 'units' if task_worker_count else 'memory-mib', 'override_instance_stats': { 'MemoryMiB': {'Min': task_ram[0], 'Max': task_ram[1]}, 'VCpuCount': {'Min': task_cpu[0], 'Max': task_cpu[1]}, From a4f3bfd31c46a42e5c74e6519d71afb6f697fad5 Mon Sep 17 00:00:00 2001 From: "Gabriele A. Ron" Date: Mon, 19 Feb 2024 23:04:44 -0600 Subject: [PATCH 06/15] Updated dependencies --- pyproject.toml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 981805f..f57ab96 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,7 @@ license = "Apache-2.0" authors = [ {name = "Nikhil Patel", email = "npatel@cars.com"} ] + keywords = [ "AWS", "EC2", @@ -19,6 +20,7 @@ keywords = [ "Cluster", "Jupyter" ] + classifiers = [ "Development Status :: 5 - Production/Stable", "Environment :: Console", @@ -34,18 +36,21 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", ] + dynamic = ["version"] dependencies = [ - "boto3~=1.19.0", - "pyyaml~=5.3.0", - "schema~=0.7.0", + "boto3", + "pyyaml", + "schema", ] + [project.optional-dependencies] test = [ "pytest~=7.1.0", "pytest-cov~=4.0" ] + dev = [ "bump2version~=1.0", ] From fb8388457b18fda75fe3b789a7053dd999021f06 Mon Sep 17 00:00:00 2001 From: "Gabriele A. Ron" Date: Wed, 21 Feb 2024 04:47:57 -0600 Subject: [PATCH 07/15] Added destroy_on_create --- src/forge/create.py | 6 ++++++ src/forge/parser.py | 1 + src/forge/yaml_loader.py | 1 + 3 files changed, 8 insertions(+) diff --git a/src/forge/create.py b/src/forge/create.py index 47ff425..30cbe2e 100755 --- a/src/forge/create.py +++ b/src/forge/create.py @@ -664,6 +664,12 @@ def search_and_create(config, task, instance_details): e = detail[0] if e['state'] in ['running', 'stopped', 'stopping', 'pending']: logger.info('%s is %s, the IP is %s', task, e['state'], e['ip']) + + if config.get('destroy_on_create'): + logger.info('destroy_on_create true, destroying fleet.') + destroy(config) + create_template(n, config, task) + create_fleet(n, config, task, instance_details) else: if len(e['fleet_id']) != 0: logger.info('Fleet is running without EC2, will recreate it.') diff --git a/src/forge/parser.py b/src/forge/parser.py index fb72e41..fb4f02a 100755 --- a/src/forge/parser.py +++ b/src/forge/parser.py @@ -127,6 +127,7 @@ def add_job_args(parser): common_grp.add_argument('--valid_time', '--valid-time', type=positive_int_arg) common_grp.add_argument('--user_data', '--user-data', nargs='*') common_grp.add_argument('--gpu', action='store_true', dest='gpu_flag') + common_grp.add_argument('--destroy_on_create', '--destroy-on-create', action='store_true', default=None) def add_action_args(parser): diff --git a/src/forge/yaml_loader.py b/src/forge/yaml_loader.py index d4042b3..e25e0a6 100755 --- a/src/forge/yaml_loader.py +++ b/src/forge/yaml_loader.py @@ -135,6 +135,7 @@ def _get_type(x): Optional('cpu'): And(list, error='Invalid CPU cores'), Optional('destroy_after_success'): And(bool), Optional('destroy_after_failure'): And(bool), + Optional('destroy_on_create'): And(bool), Optional('disk'): And(Use(int), positive_int), Optional('disk_device_name'): And(str, len, error='Invalid Device Name'), Optional('forge_env'): And(str, len, error='Invalid Environment'), From e12307616a7e2fa03ae2aa15dca7d20bd7984c33 Mon Sep 17 00:00:00 2001 From: "Gabriele A. Ron" Date: Fri, 23 Feb 2024 02:49:04 -0600 Subject: [PATCH 08/15] Fixed potential bug --- src/forge/yaml_loader.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/forge/yaml_loader.py b/src/forge/yaml_loader.py index e25e0a6..a68cbb3 100755 --- a/src/forge/yaml_loader.py +++ b/src/forge/yaml_loader.py @@ -192,7 +192,7 @@ def load_config(args): logger.info('Checking config file: %s', args['yaml']) logger.debug('Required User config is %s', user_config) - env = args['forge_env'] or user_config.get('forge_env') + env = args.get('forge_env') or user_config.get('forge_env') if env is None: logger.error("'forge_env' variable required.") @@ -219,12 +219,11 @@ def load_config(args): env_config.update(normalize_config(env_config)) check_keys(env_config['region'], env_config.get('aws_profile')) - additional_config_data = env_config.pop('additional_config', None) + additional_config_data = env_config.pop('additional_config', []) additional_config = [] - if additional_config_data: - for i in additional_config_data: - ADDITIONAL_KEYS.append(i['name']) - additional_config.append(i) + for i in additional_config_data: + ADDITIONAL_KEYS.append(i['name']) + additional_config.append(i) logger.debug('Additional config options: %s', additional_config) From ee6f9ac8f03f75b02b90325a55154607cbcac6da Mon Sep 17 00:00:00 2001 From: "Gabriele A. Ron" Date: Fri, 23 Feb 2024 02:58:26 -0600 Subject: [PATCH 09/15] Moved to get_nlist() --- src/forge/common.py | 29 +++++++++++++++++++++++++++++ src/forge/rsync.py | 15 ++------------- src/forge/run.py | 11 ++--------- src/forge/start.py | 26 ++++++-------------------- src/forge/stop.py | 27 +++++++-------------------- tests/test_start.py | 10 ++-------- 6 files changed, 48 insertions(+), 70 deletions(-) diff --git a/src/forge/common.py b/src/forge/common.py index ec65443..4fe65b4 100755 --- a/src/forge/common.py +++ b/src/forge/common.py @@ -144,6 +144,35 @@ def get_ip(details, states): return [(i['ip'], i['id']) for i in list(filter(lambda x: x['state'] in states, details))] +def get_nlist(config): + """get list of instance names based on service + + Parameters + ---------- + config : dict + Forge configuration data + + Returns + ------- + list + List of instance names + """ + date = config.get('date', '') + market = config.get('market', DEFAULT_ARG_VALS['market']) + name = config['name'] + service = config['service'] + + n_list = [] + if service == "cluster": + n_list.append(f'{name}-{market[0]}-{service}-master-{date}') + if config.get('rr_all'): + n_list.append(f'{name}-{market[-1]}-{service}-worker-{date}') + elif service == "single": + n_list.append(f'{name}-{market[0]}-{service}-{date}') + + return n_list + + @contextlib.contextmanager def key_file(secret_id, region, profile): """Safely retrieve a secret file from AWS for temporary use. diff --git a/src/forge/rsync.py b/src/forge/rsync.py index 654728e..bb62b36 100755 --- a/src/forge/rsync.py +++ b/src/forge/rsync.py @@ -6,7 +6,7 @@ from . import DEFAULT_ARG_VALS, REQUIRED_ARGS from .parser import add_basic_args, add_general_args, add_env_args, add_action_args -from .common import ec2_ip, key_file, get_ip, exit_callback +from .common import ec2_ip, key_file, get_ip, get_nlist, exit_callback logger = logging.getLogger(__name__) @@ -40,11 +40,6 @@ def rsync(config): config : dict Forge configuration data """ - name = config.get('name') - date = config.get('date', '') - market = config.get('market', DEFAULT_ARG_VALS['market']) - service = config.get('service') - rr_all = config.get('rr_all') def _rsync(config, ip): """performs the rsync to a given ip @@ -84,13 +79,7 @@ def _rsync(config, ip): else: logger.info('Rsync successful:\n%s', output) - n_list = [] - if service == "cluster": - n_list.append(f'{name}-{market[0]}-{service}-master-{date}') - if rr_all: - n_list.append(f'{name}-{market[-1]}-{service}-worker-{date}') - elif service == "single": - n_list.append(f'{name}-{market[0]}-{service}-{date}') + n_list = get_nlist(config) for n in n_list: try: diff --git a/src/forge/run.py b/src/forge/run.py index f3f465f..1f0f96d 100755 --- a/src/forge/run.py +++ b/src/forge/run.py @@ -7,7 +7,7 @@ from . import DEFAULT_ARG_VALS, REQUIRED_ARGS from .exceptions import ExitHandlerException from .parser import add_basic_args, add_general_args, add_env_args, add_action_args -from .common import ec2_ip, key_file, get_ip, destroy_hook, user_accessible_vars, FormatEmpty, exit_callback +from .common import ec2_ip, key_file, get_ip, destroy_hook, user_accessible_vars, FormatEmpty, exit_callback, get_nlist from .destroy import destroy logger = logging.getLogger(__name__) @@ -77,7 +77,6 @@ def _run(config, ip): pem_secret = config['forge_pem_secret'] region = config['region'] profile = config.get('aws_profile') - env = config['forge_env'] with key_file(pem_secret, region, profile) as pem_path: fmt = FormatEmpty() @@ -94,13 +93,7 @@ def _run(config, ip): ) return exc.returncode - n_list = [] - if service == "cluster": - n_list.append(f'{name}-{market[0]}-{service}-master-{date}') - if rr_all: - n_list.append(f'{name}-{market[-1]}-{service}-worker-{date}') - elif service == "single": - n_list.append(f'{name}-{market[0]}-{service}-{date}') + n_list = get_nlist(config) for n in n_list: try: diff --git a/src/forge/start.py b/src/forge/start.py index 3b18f79..8b8ff41 100755 --- a/src/forge/start.py +++ b/src/forge/start.py @@ -6,7 +6,7 @@ from . import REQUIRED_ARGS from .parser import add_basic_args, add_general_args, add_env_args -from .common import ec2_ip, get_ip, set_boto_session +from .common import ec2_ip, get_ip, set_boto_session, get_nlist logger = logging.getLogger(__name__) @@ -75,25 +75,11 @@ def start(config): config : dict Forge configuration data """ - name = config['name'] - date = config.get('date', '') - service = config['service'] market = config.get('market') - n_list = [] - if service == "cluster": - if market[0] == 'spot': - logger.error('Master is a spot instance; you cannot start a spot instance') - elif market[0] == 'on-demand': - n_list.append(f'{name}-{market[0]}-{service}-master-{date}') - - if market[-1] == 'spot': - logger.error('Worker is a spot fleet; you cannot start a spot fleet') - elif market[-1] == 'on-demand': - n_list.append(f'{name}-{market[0]}-{service}-worker-{date}') - elif service == "single": - if market[0] == 'spot': - logger.error('The instance is a spot instance; you cannot start a spot instance') - elif market[0] == 'on-demand': - n_list.append(f'{name}-{market[0]}-{service}-{date}') + if 'spot' in market: + logger.error('Master or worker is a spot instance; you cannot start a spot instance') + # sys.exit(1) # ToDo: Should we change the tests to reflect an exit or allow it to continue? + + n_list = get_nlist({**config, 'rr_all': True}) start_fleet(n_list, config) diff --git a/src/forge/stop.py b/src/forge/stop.py index 7426a21..7888a40 100755 --- a/src/forge/stop.py +++ b/src/forge/stop.py @@ -6,7 +6,7 @@ from . import REQUIRED_ARGS from .parser import add_basic_args, add_general_args, add_env_args -from .common import ec2_ip, get_ip, set_boto_session +from .common import ec2_ip, get_ip, set_boto_session, get_nlist logger = logging.getLogger(__name__) @@ -70,25 +70,12 @@ def stop(config): config : dict Forge configuration data """ - name = config['name'] - date = config.get('date', '') - service = config['service'] market = config.get('market') - n_list = [] - if service == "cluster": - if market[0] == 'spot': - logger.error('Master is a spot instance; you cannot stop a spot instance') - elif market[0] == 'on-demand': - n_list.append(f'{name}-{market[0]}-{service}-master-{date}') - - if market[-1] == 'spot': - logger.error('Worker is a spot fleet; you cannot stop a spot fleet') - elif market[-1] == 'on-demand': - n_list.append(f'{name}-{market[0]}-{service}-worker-{date}') - elif service == "single": - if market[0] == 'spot': - logger.error('The instance is a spot instance; you cannot stop a spot instance') - elif market[0] == 'on-demand': - n_list.append(f'{name}-{market[0]}-{service}-{date}') + if 'spot' in market: + logger.error('Master or worker is a spot instance; you cannot stop a spot instance') + # sys.exit(1) # ToDo: Should we change the tests to reflect an exit or allow it to continue? + + n_list = get_nlist({**config, 'rr_all': True}) + stop_fleet(n_list, config) diff --git a/tests/test_start.py b/tests/test_start.py index 9c0abd5..c3513ac 100644 --- a/tests/test_start.py +++ b/tests/test_start.py @@ -55,14 +55,8 @@ def test_start_error_in_spot_instance(mock_start_fleet, caplog, service, markets "service": service, } error_msg = "" - if service == "cluster": - if markets[0] == "spot": - error_msg = "Master is a spot instance; you cannot start a spot instance" - elif markets[1] == "spot": - error_msg = "Worker is a spot fleet; you cannot start a spot fleet" - else: - if markets[0] == "spot": - error_msg = "The instance is a spot instance; you cannot start a spot instance" + if 'spot' in markets: + error_msg = 'Master or worker is a spot instance; you cannot start a spot instance' with caplog.at_level(logging.ERROR): start.start(config) From d1f44c09bafdb31a106d3cdf466f3c91abc64f47 Mon Sep 17 00:00:00 2001 From: "Gabriele A. Ron" Date: Mon, 26 Feb 2024 14:15:09 -0600 Subject: [PATCH 10/15] Version 1.2.0 --- .bumpversion.cfg | 2 +- CHANGELOG.md | 5 +++++ pyproject.toml | 5 +++-- src/forge/__init__.py | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 32da4e3..c36372f 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.1.1 +current_version = 1.2.0 commit = True tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5a4a19..8fe8472 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased] +## [1.2.0] - 2024-02-26 +- **Create** - Added `destroy_on_create` +- **Common** - Moved all `n_list` functions to `get_nlist()` +- **Dependencies** - Updated dependencies and tested on latest versions + ## [1.1.1] - 2024-02-08 ### Changed diff --git a/pyproject.toml b/pyproject.toml index 8e060eb..7719751 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ authors = [ {name = "Gabriele Ron", email = "gron@cars.com"}, {name = "Joao Moreira", email = "jmoreira@cars.com"} ] + maintainers = [ {name = "Nikhil Patel", email = "npatel@cars.com"}, {name = "Gabriele Ron", email = "gron@cars.com"}, @@ -54,8 +55,8 @@ dependencies = [ [project.optional-dependencies] test = [ - "pytest~=7.1.0", - "pytest-cov~=4.0" + "pytest", + "pytest-cov" ] dev = [ diff --git a/src/forge/__init__.py b/src/forge/__init__.py index 1781885..3e1d924 100755 --- a/src/forge/__init__.py +++ b/src/forge/__init__.py @@ -1,4 +1,4 @@ -__version__ = "1.1.1" +__version__ = "1.2.0" # Default values for forge's essential arguments DEFAULT_ARG_VALS = { From 751eea266307a021275b6330dadbae1bc03847e1 Mon Sep 17 00:00:00 2001 From: "Gabriele A. Ron" Date: Wed, 28 Feb 2024 11:52:02 -0600 Subject: [PATCH 11/15] Add create_timeout configuration option --- CHANGELOG.md | 1 + src/forge/__init__.py | 3 ++- src/forge/configure.py | 1 + src/forge/create.py | 6 ++++++ 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fe8472..9378ed8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [1.2.0] - 2024-02-26 - **Create** - Added `destroy_on_create` +- **Create** - Added `create_timeout` option - **Common** - Moved all `n_list` functions to `get_nlist()` - **Dependencies** - Updated dependencies and tested on latest versions diff --git a/src/forge/__init__.py b/src/forge/__init__.py index 3e1d924..69f845e 100755 --- a/src/forge/__init__.py +++ b/src/forge/__init__.py @@ -12,7 +12,8 @@ 'default_ratio': [8, 8], 'valid_time': 8, 'ec2_max': 768, - 'spot_strategy': 'price-capacity-optimized' + 'spot_strategy': 'price-capacity-optimized', + 'create_timeout': 600 } # Required arguments for each Forge job diff --git a/src/forge/configure.py b/src/forge/configure.py index e2f58fb..ccc5578 100755 --- a/src/forge/configure.py +++ b/src/forge/configure.py @@ -75,6 +75,7 @@ def check_env_yaml(env_yaml): error='Invalid spot allocation strategy'), Optional('on_demand_failover'): And(bool), Optional('spot_retries'): And(Use(int), lambda x: x > 0), + Optional('create_timeout'): And(Use(int), lambda x: x > 0), }) try: validated = schema.validate(env_yaml) diff --git a/src/forge/create.py b/src/forge/create.py index 30cbe2e..2fd354a 100755 --- a/src/forge/create.py +++ b/src/forge/create.py @@ -167,6 +167,12 @@ def create_status(n, request, config): create_time = fleet_details.get('CreateTime') time_without_spot = 0 while current_status != 'fulfilled': + if t > DEFAULT_ARG_VALS['create_timeout']: + logger.error('Timeout of %s seconds hit for instance fulfillment; Aborting.', DEFAULT_ARG_VALS['create_timeout']) + if destroy_flag: + destroy(config) + exit_callback(config, exit=True) + if current_status == 'pending_fulfillment': time.sleep(10) t += 10 From 891ce4bd778559405a067aa6d6b2f25f044bd572 Mon Sep 17 00:00:00 2001 From: "Gabriele A. Ron" Date: Wed, 28 Feb 2024 11:52:56 -0600 Subject: [PATCH 12/15] Remove default create_timeout setting --- src/forge/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/forge/__init__.py b/src/forge/__init__.py index 69f845e..3e1d924 100755 --- a/src/forge/__init__.py +++ b/src/forge/__init__.py @@ -12,8 +12,7 @@ 'default_ratio': [8, 8], 'valid_time': 8, 'ec2_max': 768, - 'spot_strategy': 'price-capacity-optimized', - 'create_timeout': 600 + 'spot_strategy': 'price-capacity-optimized' } # Required arguments for each Forge job From 4c276d8dcb2a96ab6aee21f0682855181a0f95cb Mon Sep 17 00:00:00 2001 From: "Gabriele A. Ron" Date: Wed, 28 Feb 2024 12:04:52 -0600 Subject: [PATCH 13/15] Fix create_timeout check --- src/forge/create.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/forge/create.py b/src/forge/create.py index 2fd354a..09ed44e 100755 --- a/src/forge/create.py +++ b/src/forge/create.py @@ -167,8 +167,8 @@ def create_status(n, request, config): create_time = fleet_details.get('CreateTime') time_without_spot = 0 while current_status != 'fulfilled': - if t > DEFAULT_ARG_VALS['create_timeout']: - logger.error('Timeout of %s seconds hit for instance fulfillment; Aborting.', DEFAULT_ARG_VALS['create_timeout']) + if config.get('create_timeout') and t > config['create_timeout']: + logger.error('Timeout of %s seconds hit for instance fulfillment; Aborting.', config['create_timeout']) if destroy_flag: destroy(config) exit_callback(config, exit=True) From d43ee3cc93ae279303ae6a8c9acb0ba43c6ac7e3 Mon Sep 17 00:00:00 2001 From: "Gabriele A. Ron" Date: Fri, 1 Mar 2024 08:44:33 -0600 Subject: [PATCH 14/15] Reduce version to 1.1.0 --- .bumpversion.cfg | 2 +- CHANGELOG.md | 11 ++--------- src/forge/__init__.py | 2 +- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index c36372f..73dded6 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.2.0 +current_version = 1.1.0 commit = True tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9378ed8..1c5b1a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,20 +6,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased] -## [1.2.0] - 2024-02-26 +## [1.1.0] - 2024-02-26 +### Added - **Create** - Added `destroy_on_create` - **Create** - Added `create_timeout` option - **Common** - Moved all `n_list` functions to `get_nlist()` - **Dependencies** - Updated dependencies and tested on latest versions - -## [1.1.1] - 2024-02-08 - -### Changed - **Create** - Set default boto3 session at beginning of create to resolve region bug - -## [1.1.0] - 2024-02-07 - -### Added - **Create** - Multi-AZ functionality - Spot retries diff --git a/src/forge/__init__.py b/src/forge/__init__.py index 3e1d924..a96e573 100755 --- a/src/forge/__init__.py +++ b/src/forge/__init__.py @@ -1,4 +1,4 @@ -__version__ = "1.2.0" +__version__ = "1.1.0" # Default values for forge's essential arguments DEFAULT_ARG_VALS = { From f503e30c4fdeda8bd55baf1078e2c32641a2ffe1 Mon Sep 17 00:00:00 2001 From: "Gabriele A. Ron" Date: Fri, 5 Apr 2024 15:39:13 -0500 Subject: [PATCH 15/15] Remove Hacktoberfest 2022 branding --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 39d80bc..3ab4694 100755 --- a/README.md +++ b/README.md @@ -2,7 +2,6 @@ [![GitHub license](https://img.shields.io/github/license/carsdotcom/cars-forge?color=navy&label=License&logo=License&style=flat-square)](https://github.com/carsdotcom/cars-forge/blob/main/LICENSE) [![PyPI](https://img.shields.io/pypi/v/cars-forge?color=navy&style=flat-square)](https://pypi.org/project/cars-forge/) -![hacktoberfest](https://img.shields.io/github/issues/carsdotcom/cars-forge?color=orange&label=Hacktoberfest%202022&style=flat-square&?labelColor=black) ![PyPI - Downloads](https://img.shields.io/pypi/dm/cars-forge?color=navy&style=flat-square) ![GitHub Workflow Status (branch)](https://img.shields.io/github/workflow/status/carsdotcom/cars-forge/Publish%20Package/main?color=navy&style=flat-square) ![GitHub contributors](https://img.shields.io/github/contributors/carsdotcom/cars-forge?color=navy&style=flat-square)