diff --git a/.circleci/config.yml b/.circleci/config.yml index 35193f8..d718a3b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,7 +9,7 @@ jobs: - run: name: pip env command: | - sudo pip install pipenv + sudo pip install pipenv==2022.4.8 pipenv install - run: name: install dependencies diff --git a/CHANGELOG.md b/CHANGELOG.md index d9a1ad2..0c148dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,15 @@ Documenting All Changes to the Skelebot Project --- +## v1.29.0 +#### Changed +- **CodeArtifact Dependencies** | Adds an option to pull CodeArtifact Python packages into a libs folder for install during docker build + +--- + ## v1.28.0 +#### Merged: 2022-04-15 +#### Released: 2022-04-15 #### Changed - **In Memory Pull** | Allows S3Repo class to pull artifacts and return them directly in python diff --git a/VERSION b/VERSION index 3c71e47..5e57fb8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.28.0 \ No newline at end of file +1.29.0 diff --git a/docs/dependencies.md b/docs/dependencies.md index 499ae19..47f8436 100644 --- a/docs/dependencies.md +++ b/docs/dependencies.md @@ -42,6 +42,23 @@ dependencies: - github:myGitHub/fakeRepo ``` +### CodeArtifact Python Packages + +Skelebot also supports pulling Python packages that are stored in AWS CodeArtifact. This requires a good deal of information in order to authenticate and pull the correct asset for the package. + +``` +language: Python +dependencies: +- ca_file:{domain}:{owner}:{repo}:{pkg}:{version}:{profile} +``` +These values are needed in order to preperly obtain the package and include it in the Docker image. +- domain - The domain name of the AWS CodeArtifact repository (ex: my_domain) +- owner - The owner of the AWS CodeArtifact repository (ex: 111122223333) +- repo - The repository in CodeArtifact where the package is located (ex: my_repo) +- pkg - The name of the Python package to be installed (ex: my_package) +- version - The version of the package to be pulled (ex: 1.0.0) +- profile - [OPTIONAL] The AWS profile on your machine that has access to this CodeArtifact repository (ex: dev) + NOTES: - When installing via `file:` or `github:` the ability to specify a version is not available. diff --git a/skelebot/systems/generators/dockerfile.py b/skelebot/systems/generators/dockerfile.py index 463de0c..71dc72a 100644 --- a/skelebot/systems/generators/dockerfile.py +++ b/skelebot/systems/generators/dockerfile.py @@ -2,10 +2,12 @@ import os import re +from subprocess import call from ..execution import commandBuilder FILE_PATH = "{path}/Dockerfile" +PY_DOWNLOAD_CA = "aws codeartifact get-package-version-asset --domain {domain} --domain-owner {owner} --repository {repo} --package {pkg} --package-version {version}{profile} --format pypi --asset {asset} libs/{asset}" PY_INSTALL = "RUN [\"pip\", \"install\", \"{dep}\"]\n" PY_INSTALL_VERSION = "RUN [\"pip\", \"install\", \"{depName}=={version}\"]\n" PY_INSTALL_GITHUB = "RUN [\"pip\", \"install\", \"git+{depPath}\"]\n" @@ -46,13 +48,27 @@ def buildDockerfile(config): # Add language dependencies if (config.language == "Python"): for dep in config.dependencies: - depSplit = dep.split(":", maxsplit=1) - if ("github:" in dep): - docker += PY_INSTALL_GITHUB.format(depPath=depSplit[1]) - elif ("file:" in dep): + depSplit = dep.split(":") + if (dep.startswith("github:")): + docker += PY_INSTALL_GITHUB.format(depPath=dep.split(":", maxsplit=1)[1]) + elif (dep.startswith("file:")): docker += PY_INSTALL_FILE.format(depPath=depSplit[1]) - elif ("req:" in dep): + elif (dep.startswith("req:")): docker += PY_INSTALL_REQ.format(depPath=depSplit[1]) + elif (dep.startswith("ca_file:")): + domain = depSplit[1] + owner = depSplit[2] + repo = depSplit[3] + pkg = depSplit[4] + version = depSplit[5] + asset = f"{pkg.replace('-', '_')}-{version}-py3-none-any.whl" + profile = f" --profile {depSplit[6]}" if (len(depSplit) > 6) else "" + cmd = PY_DOWNLOAD_CA.format(domain=domain, owner=owner, repo=repo, pkg=pkg, version=version, asset=asset, profile=profile) + status = call(cmd, shell=True) + if (status != 0): + raise Exception("Failed to Obtain CodeArtifact Package") + + docker += PY_INSTALL_FILE.format(depPath=f"libs/{asset}") # if using PIP version specifiers, will be handled as a standard case elif dep.count("=") == 1 and not re.search(r"[!<>~]", dep): verSplit = dep.split("=") @@ -62,9 +78,9 @@ def buildDockerfile(config): if (config.language == "R"): for dep in config.dependencies: depSplit = dep.split(":") - if ("github:" in dep): + if (dep.startswith("github:")): docker += R_INSTALL_GITHUB.format(depPath=depSplit[1], depName=depSplit[2]) - elif ("file:" in dep): + elif (dep.startswith("file:")): docker += R_INSTALL_FILE.format(depPath=depSplit[1], depName=depSplit[2]) elif ("=" in dep): verSplit = dep.split("=") @@ -74,11 +90,25 @@ def buildDockerfile(config): if (config.language == "R+Python"): for dep in config.dependencies["Python"]: - depSplit = dep.split(":", maxsplit=1) - if ("github:" in dep): - docker += PY_R_INSTALL_GITHUB.format(depPath=depSplit[1]) - elif ("file:" in dep): + depSplit = dep.split(":") + if (dep.startswith("github:")): + docker += PY_R_INSTALL_GITHUB.format(depPath=dep.split(":", maxsplit=1)[1]) + elif (dep.startswith("file:")): docker += PY_R_INSTALL_FILE.format(depPath=depSplit[1]) + elif (dep.startswith("ca_file:")): + domain = depSplit[1] + owner = depSplit[2] + repo = depSplit[3] + pkg = depSplit[4] + version = depSplit[5] + asset = f"{pkg.replace('-', '_')}-{version}-py3-none-any.whl" + profile = f" --profile {depSplit[6]}" if (len(depSplit) > 6) else "" + cmd = PY_DOWNLOAD_CA.format(domain=domain, owner=owner, repo=repo, pkg=pkg, version=version, asset=asset, profile=profile) + status = call(cmd, shell=True) + if (status != 0): + raise Exception("Failed to Obtain CodeArtifact Package") + + docker += PY_R_INSTALL_FILE.format(depPath=f"libs/{asset}") # if using PIP version specifiers, will be handled as a standard case elif dep.count("=") == 1 and not re.search(r"[!<>~]", dep): verSplit = dep.split("=") @@ -87,9 +117,9 @@ def buildDockerfile(config): docker += PY_R_INSTALL.format(dep=dep) for dep in config.dependencies["R"]: depSplit = dep.split(":") - if ("github:" in dep): + if (dep.startswith("github:")): docker += R_INSTALL_GITHUB.format(depPath=depSplit[1], depName=depSplit[2]) - elif ("file:" in dep): + elif (dep.startswith("file:")): docker += R_INSTALL_FILE.format(depPath=depSplit[1], depName=depSplit[2]) elif ("=" in dep): verSplit = dep.split("=") diff --git a/test/test_systems_generators_dockerfile.py b/test/test_systems_generators_dockerfile.py index 21be678..625070e 100644 --- a/test/test_systems_generators_dockerfile.py +++ b/test/test_systems_generators_dockerfile.py @@ -1,3 +1,9 @@ +#try: + #self.artifactory.execute(config, args) + #self.fail("Exception Not Thrown") +#except RuntimeError as err: + #self.assertEqual(str(err), "No Compatible Version Found") + import os import unittest from unittest import mock @@ -208,19 +214,42 @@ def test_buildDockerfile_cmd_path(self, mock_getcwd, mock_expanduser): self.assertTrue(data is not None) self.assertEqual(data, expectedDockerfile) + @mock.patch('skelebot.systems.generators.dockerfile.call') + @mock.patch('os.path.expanduser') + @mock.patch('os.getcwd') + def test_buildDockerfile_py_ca_file_error(self, mock_getcwd, mock_expanduser, mock_call): + folderPath = "{path}/test/files".format(path=self.path) + filePath = "{folder}/Dockerfile".format(folder=folderPath) + + mock_expanduser.return_value = "{path}/test/plugins".format(path=self.path) + mock_getcwd.return_value = folderPath + mock_call.return_value = 1 + config = sb.systems.generators.yaml.loadConfig() + config.language = "Python" + config.dependencies.append("ca_file:cars:12345:python-pkg:ml-lib:0.1.0:prod") + + try: + sb.systems.generators.dockerfile.buildDockerfile(config) + self.fail("Exception Not Thrown") + except Exception as exc: + self.assertEqual(str(exc), "Failed to Obtain CodeArtifact Package") + + @mock.patch('skelebot.systems.generators.dockerfile.call') @mock.patch('os.path.expanduser') @mock.patch('os.getcwd') - def test_buildDockerfile_base_py(self, mock_getcwd, mock_expanduser): + def test_buildDockerfile_base_py(self, mock_getcwd, mock_expanduser, mock_call): folderPath = "{path}/test/files".format(path=self.path) filePath = "{folder}/Dockerfile".format(folder=folderPath) mock_expanduser.return_value = "{path}/test/plugins".format(path=self.path) mock_getcwd.return_value = folderPath + mock_call.return_value = 0 config = sb.systems.generators.yaml.loadConfig() config.language = "Python" config.dependencies.append("github:github.com/repo") config.dependencies.append("github:https://github.com/securerepo") config.dependencies.append("file:libs/proj") + config.dependencies.append("ca_file:cars:12345:python-pkg:ml-lib:0.1.0:prod") config.dependencies.append("req:requirements.txt") config.dependencies.append("dtable=9.0") @@ -240,6 +269,8 @@ def test_buildDockerfile_base_py(self, mock_getcwd, mock_expanduser): RUN ["pip", "install", "git+https://github.com/securerepo"] COPY libs/proj libs/proj RUN ["pip", "install", "/app/libs/proj"] +COPY libs/ml_lib-0.1.0-py3-none-any.whl libs/ml_lib-0.1.0-py3-none-any.whl +RUN ["pip", "install", "/app/libs/ml_lib-0.1.0-py3-none-any.whl"] COPY requirements.txt requirements.txt RUN ["pip", "install", "-r", "/app/requirements.txt"] RUN ["pip", "install", "dtable==9.0"] @@ -248,11 +279,14 @@ def test_buildDockerfile_base_py(self, mock_getcwd, mock_expanduser): RUN rm -rf dist/ CMD /bin/bash -c \"bash build.sh --env local --log info\"\n""" + expectedCMD = "aws codeartifact get-package-version-asset --domain cars --domain-owner 12345 --repository python-pkg --package ml-lib --package-version 0.1.0 --profile prod --format pypi --asset ml_lib-0.1.0-py3-none-any.whl libs/ml_lib-0.1.0-py3-none-any.whl" + sb.systems.generators.dockerfile.buildDockerfile(config) data = None with open(filePath, "r") as file: data = file.read() + mock_call.assert_called_with(expectedCMD, shell=True) self.assertTrue(data is not None) self.assertEqual(data, expectedDockerfile) @@ -447,14 +481,16 @@ def test_buildDockerfile_custom(self, mock_getcwd, mock_expanduser): self.assertTrue(data is not None) self.assertEqual(data, expectedDockerfile) + @mock.patch('skelebot.systems.generators.dockerfile.call') @mock.patch('os.path.expanduser') @mock.patch('os.getcwd') - def test_buildDockerfile_R_plus_Python(self, mock_getcwd, mock_expanduser): + def test_buildDockerfile_R_py_ca_file_error(self, mock_getcwd, mock_expanduser, mock_call): folderPath = "{path}/test/files".format(path=self.path) filePath = "{folder}/Dockerfile".format(folder=folderPath) mock_expanduser.return_value = "{path}/test/plugins".format(path=self.path) mock_getcwd.return_value = folderPath + mock_call.return_value = -1 config = sb.systems.generators.yaml.loadConfig() config.language = "R+Python" config.dependencies = { @@ -462,6 +498,41 @@ def test_buildDockerfile_R_plus_Python(self, mock_getcwd, mock_expanduser): "numpy", "pandas", "github:github.com/repo", "github:https://github.com/securerepo", "file:libs/proj", + "ca_file:cars:12345:python-pkg:ml-lib:0.1.0:prod", + "dtable>=9.0", "dtable=9.0" + ], + "R":[ + "data.table", "here", + "github:github.com/repo:cool-lib", + "file:libs/proj:cool-proj", + "dtable=9.0" + ] + } + + try: + sb.systems.generators.dockerfile.buildDockerfile(config) + self.fail("Exception Not Thrown") + except Exception as exc: + self.assertEqual(str(exc), "Failed to Obtain CodeArtifact Package") + + @mock.patch('skelebot.systems.generators.dockerfile.call') + @mock.patch('os.path.expanduser') + @mock.patch('os.getcwd') + def test_buildDockerfile_R_plus_Python(self, mock_getcwd, mock_expanduser, mock_call): + folderPath = "{path}/test/files".format(path=self.path) + filePath = "{folder}/Dockerfile".format(folder=folderPath) + + mock_expanduser.return_value = "{path}/test/plugins".format(path=self.path) + mock_getcwd.return_value = folderPath + mock_call.return_value = 0 + config = sb.systems.generators.yaml.loadConfig() + config.language = "R+Python" + config.dependencies = { + "Python":[ + "numpy", "pandas", + "github:github.com/repo", "github:https://github.com/securerepo", + "file:libs/proj", + "ca_file:cars:12345:python-pkg:ml-lib:0.1.0:prod", "dtable>=9.0", "dtable=9.0" ], "R":[ @@ -486,6 +557,8 @@ def test_buildDockerfile_R_plus_Python(self, mock_getcwd, mock_expanduser): RUN ["pip3", "install", "git+https://github.com/securerepo"] COPY libs/proj libs/proj RUN ["pip3", "install", "/app/libs/proj"] +COPY libs/ml_lib-0.1.0-py3-none-any.whl libs/ml_lib-0.1.0-py3-none-any.whl +RUN ["pip3", "install", "/app/libs/ml_lib-0.1.0-py3-none-any.whl"] RUN ["pip3", "install", "dtable>=9.0"] RUN ["pip3", "install", "dtable==9.0"] RUN ["Rscript", "-e", "install.packages('data.table', repo='https://cloud.r-project.org'); library(data.table)"] @@ -502,11 +575,14 @@ def test_buildDockerfile_R_plus_Python(self, mock_getcwd, mock_expanduser): CMD /bin/bash -c "/./krb/init.sh user && bash build.sh --env local --log info\"\n""" sb.systems.generators.dockerfile.buildDockerfile(config) + expectedCMD = "aws codeartifact get-package-version-asset --domain cars --domain-owner 12345 --repository python-pkg --package ml-lib --package-version 0.1.0 --profile prod --format pypi --asset ml_lib-0.1.0-py3-none-any.whl libs/ml_lib-0.1.0-py3-none-any.whl" + data = None with open(filePath, "r") as file: data = file.read() self.assertTrue(data is not None) self.assertEqual(data, expectedDockerfile) + mock_call.assert_called_with(expectedCMD, shell=True) @mock.patch('os.path.expanduser') @mock.patch('os.getcwd')