add support for nomad

ucbrise · Oct 16, 2019 · aa3aedf · aa3aedf
1 parent 869d21f
commit aa3aedf
Show file tree

Hide file tree

Showing 11 changed files with 637 additions and 73 deletions.
diff --git a/clipper_admin/clipper_admin/__init__.py b/clipper_admin/clipper_admin/__init__.py
@@ -2,6 +2,8 @@
 
 from .docker.docker_container_manager import DockerContainerManager
 from .kubernetes.kubernetes_container_manager import KubernetesContainerManager
+from .nomad.nomad_container_manager import NomadContainerManager
+from .nomad.consul_dns import ConsulDNS
 from .clipper_admin import *
 from . import deployers
 from .version import __version__, __registry__

diff --git a/clipper_admin/clipper_admin/nomad/consulDNS.py b/clipper_admin/clipper_admin/nomad/consulDNS.py
diff --git a/clipper_admin/clipper_admin/nomad/consul_dns.py b/clipper_admin/clipper_admin/nomad/consul_dns.py
@@ -0,0 +1,18 @@
+from .dns import DNS
+import dns.resolver
+import socket
+
+class ConsulDNS(DNS):
+
+    def resolveSRV(self, job_name):
+        addr = '{}.service.consul'.format(job_name)
+        srv_records= dns.resolver.query(addr, 'SRV')
+        srvInfo = {}
+        for srv in srv_records:
+            srvInfo['host']     = str(srv.target).rstrip('.')
+            srvInfo['port']     = srv.port
+        host = srvInfo['host']
+        port = srvInfo['port']
+        print(host)
+        print(port)
+        return (socket.gethostbyname(addr), port)
diff --git a/clipper_admin/clipper_admin/nomad/dns.py b/clipper_admin/clipper_admin/nomad/dns.py
@@ -1,4 +1,5 @@
-
+import abc
+from abc import abstractmethod
 class DNS(abc.ABC):
 
     @abstractmethod

diff --git a/clipper_admin/clipper_admin/nomad/mgmt_deployment.py b/clipper_admin/clipper_admin/nomad/mgmt_deployment.py
@@ -0,0 +1,62 @@
+from .utils import nomad_job_prefix
+import os
+
+def mgmt_job_prefix(cluster_name):
+    return '{}-mgmt'.format(nomad_job_prefix(cluster_name))
+
+""" Nomad payload to deploy a new mgmt """
+def mgmt_deployment(job_id, datacenters, cluster_name, image, redis_ip, redis_port, num_replicas):
+    job = { 'Job': {
+        'ID': job_id,
+        'Datacenters': datacenters,
+        'Type': 'service',
+        'TaskGroups': [
+            {
+                'Name': nomad_job_prefix(cluster_name),
+                'Count': num_replicas,
+                'Tasks': [
+                    {
+                        'Name': mgmt_job_prefix(cluster_name),
+                        'Driver': 'docker',
+                        'Config': {
+                            'args': [
+                                "--redis_ip={}".format(redis_ip or os.environ('REDIS_SERVICE_IP')), # If redis_service_host == None, default to env var
+                                "--redis_port={}".format(redis_port or os.environ('REDIS_SERVICE_PORT') or True)
+                                ],
+                            'image': image,
+                            'port_map': [
+                                {'http': 1338}     
+                                ]
+                            },
+                        'Resources': {
+                            'CPU': 500,
+                            'MemoryMB': 256,
+                            'Networks': [
+                                {
+                                    'DynamicPorts': [{'Label': 'http', 'Value': 1338}]
+                                }
+                            ]
+                        },
+                        'Services': [
+                            {
+                                'Name': '{}-mgmt'.format(nomad_job_prefix(cluster_name)),
+                                'Tags': ['machine-learning', 'model', 'clipper', 'mgmt'],
+                                'PortLabel': 'http',
+                                'Checks': [
+                                    {
+                                        'Name': 'alive',
+                                        'Type': 'tcp',
+                                        'interval': 1000000000000,
+                                        'timeout': 20000000000
+                                        }
+                                    ]
+                                }
+                            ]
+                        }
+                    ]
+                }
+            ]
+
+        }
+    }
+    return job
diff --git a/clipper_admin/clipper_admin/nomad/model_deployment.py b/clipper_admin/clipper_admin/nomad/model_deployment.py
@@ -0,0 +1,71 @@
+
+from .utils import nomad_job_prefix
+
+def model_job_prefix(cluster_name):
+    return '{}-model'.format(nomad_job_prefix(cluster_name))
+
+def generate_model_job_name(cluster_name, model_name, model_version):
+    return '{}-{}-{}'.format(model_job_prefix(cluster_name), model_name, model_version)
+
+def model_check_name(cluster_name, name, version):
+    return '{}-model-{}-{}'.format(nomad_job_prefix(cluster_name), name, version)
+
+""" Nomad payload to deploy a new model """
+def model_deployment(job_id, datacenters, cluster_name, name, version, input_type, image, num_replicas):
+    job = {
+        'Job': {
+        'ID': job_id,
+        'Datacenters': datacenters,
+        'Type': 'service',
+        'TaskGroups': [
+            {
+                'Name': 'clipper-{}'.format(cluster_name),
+                'Count': num_replicas,
+                'Tasks': [
+                    {
+                    'Name': generate_model_job_name(cluster_name, name, version),
+                    'Driver': 'docker',
+                    'Env': {
+                        'CLIPPER_MODEL_NAME': name,
+                        'CLIPPER_MODEL_VERSION': version
+                    },
+                    'Config': {
+                        'image': image,
+                        'port_map': [
+                            {'zeromq': 1390}
+                        ]
+                    },
+                    'Resources': {
+                        'CPU': 500,
+                        'MemoryMB': 256,
+                        'Networks': [
+                            {
+                                'DynamicPorts': [
+                                    {'Label': 'zeromq', 'Value': 1390}
+                                 ]
+                            }
+                        ]
+                    },
+                    'Services': [
+                        {
+                            'Name': model_check_name(cluster_name, name, version),
+                            'Tags': ['machine-learning', 'model', 'clipper', name],
+                            'PortLabel': 'zeromq',
+                            'Checks': [
+                                {
+                                    'Name': 'alive',
+                                    'Type': 'tcp',
+                                    'interval': 1000000000000,
+                                    'timeout': 20000000000
+                                }
+                            ]
+                        }
+                    ]
+                    }
+                ]
+            }
+        ]
+
+        }
+    }
+    return job