-
-
-
-
-
-
-
-
-
- {{l.container_config.Cmd.join(" ")}}
-
-
-
+
+
diff --git a/docker-compose.yml b/docker-compose.yml
index 0f394a3cd..ad72d8a65 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -40,4 +40,4 @@ controller:
volumes_from:
- media
ports:
- - "8080:8080"
+ - "8080:8080"
\ No newline at end of file
diff --git a/registry.go b/registry.go
index b65de83cc..6aac5fb93 100644
--- a/registry.go
+++ b/registry.go
@@ -1,18 +1,29 @@
package shipyard
import (
- registry "github.com/shipyard/shipyard/registry/v1"
+ "crypto/tls"
+ registry "github.com/shipyard/shipyard/registry/v2"
+ "strings"
)
type Registry struct {
ID string `json:"id,omitempty" gorethink:"id,omitempty"`
Name string `json:"name,omitempty" gorethink:"name,omitempty"`
- Addr string `json:"addr,omitempty", gorethink:"addr,omitempty"`
+ Addr string `json:"addr,omitempty" gorethink:"addr,omitempty"`
+ Username string `json:"username,omitempty" gorethink:"username,omitempty"`
+ Password string `json:"password,omitempty" gorethink:"password,omitempty"`
+ TlsSkipVerify bool `json:"tls_skip_verify,omitempty" gorethink:"tls_skip_verify,omitempty"`
registryClient *registry.RegistryClient `json:"-" gorethink:"-"`
}
-func NewRegistry(id, name, addr string) (*Registry, error) {
- rClient, err := registry.NewRegistryClient(addr, nil)
+func NewRegistry(id, name, addr, username, password string, tls_skip_verify bool) (*Registry, error) {
+ var tlsConfig *tls.Config
+
+ if tls_skip_verify {
+ tlsConfig = &tls.Config{InsecureSkipVerify: true}
+ }
+
+ rClient, err := registry.NewRegistryClient(addr, tlsConfig, username, password)
if err != nil {
return nil, err
}
@@ -21,21 +32,48 @@ func NewRegistry(id, name, addr string) (*Registry, error) {
ID: id,
Name: name,
Addr: addr,
+ Username: username,
+ Password: password,
+ TlsSkipVerify: tls_skip_verify,
registryClient: rClient,
}, nil
}
+func (r *Registry) InitRegistryClient() error {
+ var tlsConfig *tls.Config
+
+ if r.TlsSkipVerify {
+ tlsConfig = &tls.Config{InsecureSkipVerify: true}
+ }
+
+ rClient, err := registry.NewRegistryClient(r.Addr, tlsConfig, r.Username, r.Password)
+ if err != nil {
+ return err
+ }
+
+ r.registryClient = rClient
+
+ return nil
+}
+
func (r *Registry) Repositories() ([]*registry.Repository, error) {
- res, err := r.registryClient.Search("", 1, 100)
+ res, err := r.registryClient.Search("")
if err != nil {
return nil, err
}
- return res.Results, nil
+ return res, nil
}
func (r *Registry) Repository(name string) (*registry.Repository, error) {
- return r.registryClient.Repository(name)
+ repoPath := name
+ tag := "latest"
+ parts := strings.Split(name, ":")
+ if len(parts) == 2 {
+ repoPath = parts[0]
+ tag = parts[1]
+ }
+ return r.registryClient.Repository(r.Addr, repoPath, tag)
}
func (r *Registry) DeleteRepository(name string) error {
diff --git a/registry/v2/error.go b/registry/v2/error.go
new file mode 100644
index 000000000..05631e6d8
--- /dev/null
+++ b/registry/v2/error.go
@@ -0,0 +1,15 @@
+package v2
+
+import (
+ "fmt"
+)
+
+type Error struct {
+ StatusCode int
+ Status string
+ msg string
+}
+
+func (e Error) Error() string {
+ return fmt.Sprintf("%s: %s", e.Status, e.msg)
+}
diff --git a/registry/v2/registry.go b/registry/v2/registry.go
new file mode 100644
index 000000000..b7165726b
--- /dev/null
+++ b/registry/v2/registry.go
@@ -0,0 +1,248 @@
+package v2
+
+import (
+ "bytes"
+ "crypto/tls"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io/ioutil"
+ "net"
+ "net/http"
+ "net/url"
+ "strings"
+ "time"
+
+ log "github.com/Sirupsen/logrus"
+)
+
+var (
+ ErrNotFound = errors.New("Not found")
+ defaultHTTPTimeout = 30 * time.Second
+)
+
+type RegistryClient struct {
+ URL *url.URL
+ tlsConfig *tls.Config
+ httpClient *http.Client
+ Username string
+ Password string
+}
+
+type Repo struct {
+ Namespace string
+ Repository string
+}
+
+type TagList struct {
+ Tags []string `json:"tags"`
+}
+
+func newHTTPClient(u *url.URL, tlsConfig *tls.Config, timeout time.Duration) *http.Client {
+ httpTransport := &http.Transport{
+ TLSClientConfig: tlsConfig,
+ }
+
+ httpTransport.Dial = func(proto, addr string) (net.Conn, error) {
+ return net.DialTimeout(proto, addr, timeout)
+ }
+ return &http.Client{Transport: httpTransport}
+}
+
+func NewRegistryClient(registryUrl string, tlsConfig *tls.Config, username string, password string) (*RegistryClient, error) {
+ u, err := url.Parse(registryUrl)
+ if err != nil {
+ return nil, err
+ }
+ httpClient := newHTTPClient(u, tlsConfig, defaultHTTPTimeout)
+ return &RegistryClient{
+ URL: u,
+ httpClient: httpClient,
+ tlsConfig: tlsConfig,
+ Username: username,
+ Password: password,
+ }, nil
+}
+
+func (client *RegistryClient) doRequest(method string, path string, body []byte, headers map[string]string) ([]byte, http.Header, error) {
+ b := bytes.NewBuffer(body)
+
+ req, err := http.NewRequest(method, client.URL.String()+"/v2"+path, b)
+ log.Debugf("Method: %s URL: %s", method, client.URL.String()+"/v2"+path)
+ if err != nil {
+ log.Error(err)
+ return nil, nil, err
+ }
+
+ req.SetBasicAuth(client.Username, client.Password)
+
+ req.Header.Add("Content-Type", "application/json")
+ if headers != nil {
+ for header, value := range headers {
+ req.Header.Add(header, value)
+ }
+ }
+
+ resp, err := client.httpClient.Do(req)
+ defer resp.Body.Close()
+
+ data, err := ioutil.ReadAll(resp.Body)
+
+ return data, resp.Header, nil
+}
+
+func (client *RegistryClient) Search(query string) ([]*Repository, error) {
+ type repo struct {
+ Repositories []string `json:"repositories"`
+ }
+
+ uri := fmt.Sprintf("/_catalog")
+ data, _, err := client.doRequest("GET", uri, nil, nil)
+ if err != nil {
+ log.Error(err)
+ return nil, err
+ }
+
+ res := &repo{}
+ if err := json.Unmarshal(data, &res); err != nil {
+ log.Error(err)
+ return nil, err
+ }
+
+ repos := []*Repository{}
+
+ // simple filter for list
+ for _, k := range res.Repositories {
+ if strings.Index(k, query) == 0 {
+ tl, err := client.getTags(k)
+ if err != nil {
+ log.Errorf("error getting tags: %s", err)
+ repos = append(repos, &Repository{
+ Name: k,
+ Tag: "",
+ HasProblems: true,
+ Architecture: "",
+ RegistryUrl: client.URL.String(),
+ Message: string(err.Error()),
+ })
+ // TODO: it is ok to skip, but we should provide information to client about this.
+ continue
+ }
+
+ for _, t := range tl.Tags {
+ // get the repository and append to the slice
+ r, err := client.Repository(client.URL.String(), k, t)
+ if err != nil {
+ log.Error(err)
+ }
+
+ // Add the repo even if there was an error, so that we know that it exists.
+ // The repo will just have name, tag, and mark some other fields as invalid.
+ repos = append(repos, r)
+ }
+ }
+ }
+
+ return repos, nil
+}
+
+func (client *RegistryClient) DeleteRepository(repo string) error {
+ tl, err := client.getTags(repo)
+ if err != nil {
+ return err
+ }
+
+ for _, t := range tl.Tags {
+ // remove tag
+ if err := client.DeleteTag(repo, t); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (client *RegistryClient) DeleteTag(repo string, tag string) error {
+ r, err := client.Repository(client.URL.String(), repo, tag)
+ if err != nil {
+ return err
+ }
+
+ uri := fmt.Sprintf("/%s/manifests/%s", repo, r.Digest)
+ if _, _, err := client.doRequest("DELETE", uri, nil, nil); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (client *RegistryClient) Repository(registryUrl, name, tag string) (*Repository, error) {
+ if tag == "" {
+ tag = "latest"
+ }
+
+ size := int64(0)
+
+ invalidRepository := &Repository{
+ Name: name,
+ Tag: tag,
+ HasProblems: true,
+ Architecture: "",
+ RegistryUrl: registryUrl,
+ }
+
+ uri := fmt.Sprintf("/%s/manifests/%s", name, tag)
+
+ log.Debugf("requesting manifest for %s", uri)
+ data, hdr, err := client.doRequest("GET", uri, nil, nil)
+ if err != nil {
+ invalidRepository.Message = fmt.Sprintf("Error when getting manifest for %s, error = %s", uri, err.Error())
+ log.Error(invalidRepository.Message)
+ return invalidRepository, err
+ }
+
+ repo := &Repository{}
+ if err := json.Unmarshal(data, &repo); err != nil {
+ invalidRepository.Message = fmt.Sprintf("Error when binding manifests for %s, error = %s", uri, err.Error())
+ log.Error(invalidRepository.Message)
+ return invalidRepository, err
+ }
+
+ repo.RegistryUrl = registryUrl
+ repo.Digest = hdr.Get("Docker-Content-Digest")
+ log.Debugf("Got docker content digest %s", repo.Digest)
+
+ var headers map[string]string
+ headers = make(map[string]string)
+ headers["Accept"]= "application/vnd.docker.distribution.manifest.v2+json"
+ data, hdr, err = client.doRequest("GET", uri, nil, headers)
+ ls := &Manifest{}
+ if err := json.Unmarshal(data, &ls); err != nil {
+ invalidRepository.Message = fmt.Sprintf("Error when binding manifests for %s, error = %s", uri, err.Error())
+ log.Error(invalidRepository.Message)
+ return invalidRepository, err
+ }
+ for _, i := range ls.Layers {
+ size += i.Size
+ }
+ repo.Size = size
+
+ return repo, nil
+}
+
+func (client *RegistryClient) getTags(repo string) (*TagList, error) {
+ uri := fmt.Sprintf("/%s/tags/list", repo)
+ data, _, err := client.doRequest("GET", uri, nil, nil)
+ if err != nil {
+ log.Errorf("There was an error when requesting tags for %s, error = %s", uri, err.Error())
+ return nil, err
+ }
+
+ log.Debugf("Tags received %s", string(data))
+ tl := &TagList{}
+ if err := json.Unmarshal(data, &tl); err != nil {
+ return nil, err
+ }
+
+ return tl, nil
+}
diff --git a/registry/v2/repository.go b/registry/v2/repository.go
new file mode 100644
index 000000000..50a417e8f
--- /dev/null
+++ b/registry/v2/repository.go
@@ -0,0 +1,56 @@
+package v2
+
+import ()
+
+type (
+ Tag struct {
+ Name string
+ }
+
+ FsLayer struct {
+ BlobSum string `json:"blobSum"`
+ }
+
+ JSONWebKey struct {
+ CRV string `json:"crv"`
+ KID string `json:"kid"`
+ KTY string `json:"kty"`
+ X string `json:"x"`
+ Y string `json:"y"`
+ }
+
+ Header struct {
+ JSONWebKey JSONWebKey `json:"jwk"`
+ Algorithm string `json:"alg"`
+ }
+
+ Signature struct {
+ Header Header `json:"header"`
+ Signature string `json:"signature`
+ Protected string `json:"protected"`
+ }
+
+ Layers struct {
+ Size int64 `json:"size"`
+ }
+
+ Manifest struct {
+ SchemaVersion int `json:"schemaVersion,omitempty"`
+ Layers []Layers `json:"layers"`
+ }
+
+ Repository struct {
+ SchemaVersion int `json:"schemaVersion,omitempty"`
+ Digest string `json:"digest,omitempty"`
+ Name string `json:"name"`
+ Tag string `json:"tag"`
+ Architecture string `json:"architecture"`
+ FsLayers []FsLayer `json:"fsLayers"`
+ Signatures []Signature `json:"signatures"`
+ HasProblems bool `json:"hasProblems"`
+ Message string `json:"message"`
+ RegistryUrl string `json:"registryUrl"`
+ RegistryName string `json:"registryName"`
+ Size int64 `json:"size"`
+ }
+)
\ No newline at end of file
diff --git a/utils/tlsutils/tlsutils.go b/utils/tlsutils/tlsutils.go
new file mode 100644
index 000000000..bb694ab2f
--- /dev/null
+++ b/utils/tlsutils/tlsutils.go
@@ -0,0 +1,205 @@
+package tlsutils
+
+import (
+ "bytes"
+ "crypto/rand"
+ "crypto/rsa"
+ "crypto/tls"
+ "crypto/x509"
+ "crypto/x509/pkix"
+ "encoding/pem"
+ "errors"
+ "io/ioutil"
+ "math/big"
+ "net"
+ "os"
+ "path/filepath"
+ "time"
+
+ log "github.com/Sirupsen/logrus"
+)
+
+var (
+ ErrNotRSAPrivateKey = errors.New("private key is not an RSA key")
+)
+
+const (
+ systemCertPath = "/etc/ssl"
+)
+
+func loadSystemCertificates(certPool *x509.CertPool) error {
+ if _, err := os.Stat(systemCertPath); os.IsNotExist(err) {
+ return nil
+ }
+
+ log.Debugf("loading system certificates: dir=%s", systemCertPath)
+
+ return filepath.Walk(systemCertPath, func(path string, fi os.FileInfo, err error) error {
+ if !fi.IsDir() {
+ cert, err := ioutil.ReadFile(path)
+ if err != nil {
+ return err
+ }
+
+ certPool.AppendCertsFromPEM(cert)
+ }
+ return nil
+ })
+}
+
+// GetServerTLSConfig returns a TLS config for using with ListenAndServeTLS
+// This sets up the Root and Client CAs for verification
+func GetServerTLSConfig(caCert, serverCert, serverKey []byte, allowInsecure bool) (*tls.Config, error) {
+ // TLS config
+ var tlsConfig tls.Config
+ tlsConfig.InsecureSkipVerify = allowInsecure
+ certPool := x509.NewCertPool()
+
+ // load system certs
+ if err := loadSystemCertificates(certPool); err != nil {
+ return nil, err
+ }
+
+ // append custom CA
+ certPool.AppendCertsFromPEM(caCert)
+
+ tlsConfig.RootCAs = certPool
+ tlsConfig.ClientCAs = certPool
+
+ log.Debugf("tls root CAs: %d", len(tlsConfig.RootCAs.Subjects()))
+
+ // require client auth
+ tlsConfig.ClientAuth = tls.VerifyClientCertIfGiven
+
+ // server cert
+ keypair, err := tls.X509KeyPair(serverCert, serverKey)
+ if err != nil {
+ return &tlsConfig, err
+
+ }
+ tlsConfig.Certificates = []tls.Certificate{keypair}
+
+ return &tlsConfig, nil
+}
+
+func newCertificate(org string) (*x509.Certificate, error) {
+ now := time.Now()
+ // need to set notBefore slightly in the past to account for time
+ // skew in the VMs otherwise the certs sometimes are not yet valid
+ notBefore := now.Add(-time.Second*300)
+ notAfter := notBefore.Add(time.Hour * 24 * 1080)
+
+ serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
+ serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
+ if err != nil {
+ return nil, err
+
+ }
+
+ return &x509.Certificate{
+ SerialNumber: serialNumber,
+ Subject: pkix.Name{
+ Organization: []string{org},
+ },
+ NotBefore: notBefore,
+ NotAfter: notAfter,
+
+ KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageKeyAgreement,
+ BasicConstraintsValid: true,
+ }, nil
+
+}
+
+// GenerateCACertificate generates a new certificate authority from the specified org
+// and bit size and returns the certificate and key as []byte, []byte
+func GenerateCACertificate(org string, bits int) ([]byte, []byte, error) {
+ template, err := newCertificate(org)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ template.IsCA = true
+ template.KeyUsage |= x509.KeyUsageCertSign
+ template.KeyUsage |= x509.KeyUsageKeyEncipherment
+ template.KeyUsage |= x509.KeyUsageKeyAgreement
+
+ priv, err := rsa.GenerateKey(rand.Reader, bits)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ derBytes, err := x509.CreateCertificate(rand.Reader, template, template, &priv.PublicKey, priv)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ var certOut bytes.Buffer
+ var keyOut bytes.Buffer
+
+ pem.Encode(&certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
+ pem.Encode(&keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
+
+ return certOut.Bytes(), keyOut.Bytes(), nil
+}
+
+// GenerateCert generates a new certificate signed using the provided
+// certificate authority certificate and key byte arrays. It will return
+// the generated certificate and key as []byte, []byte
+func GenerateCert(hosts []string, caCert []byte, caKey []byte, org string, bits int) ([]byte, []byte, error) {
+ template, err := newCertificate(org)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ // client
+ if len(hosts) == 1 && hosts[0] == "" {
+ template.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}
+ template.KeyUsage = x509.KeyUsageDigitalSignature
+ } else { // server
+ template.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}
+ for _, h := range hosts {
+ if ip := net.ParseIP(h); ip != nil {
+ template.IPAddresses = append(template.IPAddresses, ip)
+ } else {
+ template.DNSNames = append(template.DNSNames, h)
+ }
+ }
+ }
+
+ tlsCert, err := tls.X509KeyPair(caCert, caKey)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ priv, err := rsa.GenerateKey(rand.Reader, bits)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ x509Cert, err := x509.ParseCertificate(tlsCert.Certificate[0])
+ if err != nil {
+ return nil, nil, err
+ }
+
+ derBytes, err := x509.CreateCertificate(rand.Reader, template, x509Cert, &priv.PublicKey, tlsCert.PrivateKey)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ var certOut bytes.Buffer
+ var keyOut bytes.Buffer
+
+ pem.Encode(&certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
+ pem.Encode(&keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
+
+ return certOut.Bytes(), keyOut.Bytes(), nil
+}
+
+// GetPublicKey returns the RSA public key for the specified private key
+func GetPublicKey(priv interface{}) (*rsa.PublicKey, error) {
+ if k, ok := priv.(*rsa.PrivateKey); ok {
+ return &k.PublicKey, nil
+ }
+
+ return nil, ErrNotRSAPrivateKey
+}
diff --git a/utils/tlsutils/tlsutils_test.go b/utils/tlsutils/tlsutils_test.go
new file mode 100644
index 000000000..2f4f9b94c
--- /dev/null
+++ b/utils/tlsutils/tlsutils_test.go
@@ -0,0 +1,128 @@
+package tlsutils
+
+import (
+ "crypto/tls"
+ "crypto/x509"
+ "testing"
+)
+
+const (
+ testOrg = "test-org"
+ bits = 2048
+)
+
+func TestGenerateCACertificate(t *testing.T) {
+ caCert, caKey, err := GenerateCACertificate(testOrg, bits)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if caCert == nil {
+ t.Fatalf("expected ca cert; received nil")
+ }
+
+ if caKey == nil {
+ t.Fatalf("expected ca key; received nil")
+ }
+
+ keypair, err := tls.X509KeyPair(caCert, caKey)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ c, err := x509.ParseCertificate(keypair.Certificate[0])
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if !c.IsCA {
+ t.Fatalf("expected CA; received non-CA cert")
+ }
+
+ if c.Subject.Organization[0] != testOrg {
+ t.Fatalf("expected org %s; received %s", testOrg, c.Subject.Organization[0])
+ }
+}
+
+func TestGenerateCert(t *testing.T) {
+ caCert, caKey, err := GenerateCACertificate(testOrg, bits)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if caCert == nil {
+ t.Fatalf("expected ca cert; received nil")
+ }
+
+ if caKey == nil {
+ t.Fatalf("expected ca key; received nil")
+ }
+
+ cert, key, err := GenerateCert([]string{}, caCert, caKey, testOrg, bits)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if cert == nil {
+ t.Fatalf("expected cert; received nil")
+ }
+
+ if key == nil {
+ t.Fatalf("expected key; received nil")
+ }
+
+ keypair, err := tls.X509KeyPair(cert, key)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ c, err := x509.ParseCertificate(keypair.Certificate[0])
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if c.IsCA {
+ t.Fatalf("expected non-CA; received CA cert")
+ }
+
+ if c.Subject.Organization[0] != testOrg {
+ t.Fatalf("expected org %s; received %s", testOrg, c.Subject.Organization[0])
+ }
+}
+
+func TestGetPublicKey(t *testing.T) {
+ caCert, caKey, err := GenerateCACertificate(testOrg, bits)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if caCert == nil {
+ t.Fatalf("expected ca cert; received nil")
+ }
+
+ if caKey == nil {
+ t.Fatalf("expected ca key; received nil")
+ }
+
+ cert, key, err := GenerateCert([]string{}, caCert, caKey, testOrg, bits)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if cert == nil {
+ t.Fatalf("expected cert; received nil")
+ }
+
+ if key == nil {
+ t.Fatalf("expected key; received nil")
+ }
+
+ keypair, err := tls.X509KeyPair(cert, key)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if _, err := GetPublicKey(keypair.PrivateKey); err != nil {
+ t.Fatal(err)
+ }
+}
diff --git a/utils/utils.go b/utils/utils.go
index 014a7cc3e..1e7e31e45 100644
--- a/utils/utils.go
+++ b/utils/utils.go
@@ -93,3 +93,13 @@ func GetClient(dockerUrl, tlsCaCert, tlsCert, tlsKey string, allowInsecure bool)
return client, nil
}
+
+// utility for specifying a timeout channel
+func ChanTimeout(timeoutInSeconds int) <-chan bool {
+ timeout := make(chan bool, 1)
+ go func() {
+ time.Sleep(time.Duration(timeoutInSeconds) * time.Second)
+ timeout <- true
+ }()
+ return timeout
+}