Merge pull request #36 from datacol-io/33_cloudsql

cloudsql instance through DM
datacol-io · Jul 21, 2017 · cdce3b8 · cdce3b8
2 parents 1fae288 + 74e77aa
commit cdce3b8
Show file tree

Hide file tree

Showing 10 changed files with 53 additions and 51 deletions.
diff --git a/api/server.go b/api/server.go
@@ -53,21 +53,15 @@ func newServer() *Server {
 		}
 	case "gcp":
 		var bucket, zone, projectId, projectNumber string
+
 		password = os.Getenv("DATACOL_API_KEY")
 		bucket = os.Getenv("DATACOL_BUCKET")
 		name = os.Getenv("DATACOL_STACK")
+
 		zone = os.Getenv("GCP_DEFAULT_ZONE")
 		region := os.Getenv("GCP_REGION")
-
-		projectId, err := metadata.ProjectID()
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		projectNumber, err = metadata.NumericProjectID()
-		if err != nil {
-			log.Fatal(err)
-		}
+		projectId = os.Getenv("GCP_PROJECT")
+		projectNumber = os.Getenv("GCP_PROJECT_NUMBER")
 
 		provider = &google.GCPCloud{
 			Project:        projectId,

diff --git a/cloud/aws/templates.go b/cloud/aws/templates.go
diff --git a/cloud/google/kubeconfig.go b/cloud/google/kubeconfig.go
@@ -9,7 +9,6 @@ import (
 	"os"
 	"path/filepath"
 
-	"cloud.google.com/go/compute/metadata"
 	log "github.com/Sirupsen/logrus"
 	container "google.golang.org/api/container/v1"
 )
@@ -45,12 +44,12 @@ type configOptions struct {
 }
 
 func (g *GCPCloud) K8sConfigPath() (string, error) {
-	c, err := metadata.InstanceAttributeValue("DATACOL_CLUSTER")
-	if err != nil {
-		return "", err
+	cname, ok := os.LookupEnv("DATACOL_CLUSTER")
+	if !ok {
+		return "", fmt.Errorf("DATACOL_CLUSTER env var not found")
 	}
 
-	return cacheKubeConfig(g.DeploymentName, g.Project, g.DefaultZone, c)
+	return cacheKubeConfig(g.DeploymentName, g.Project, g.DefaultZone, cname)
 }
 
 func cacheKubeConfig(sName, project, zone, cname string) (string, error) {

diff --git a/cloud/google/resources.go b/cloud/google/resources.go
@@ -103,6 +103,7 @@ func (g *GCPCloud) resourceListFromStack() (pb.Resources, error) {
 
 func (g *GCPCloud) ResourceCreate(name, kind string, params map[string]string) (*pb.Resource, error) {
 	rs := &pb.Resource{Name: name, Kind: kind}
+	params["project"] = g.Project
 
 	if kind == "postgres" {
 		if _, ok := params["tier"]; !ok {

diff --git a/cloud/google/templates.go b/cloud/google/templates.go
diff --git a/cloud/google/templates/mysql.tmpl b/cloud/google/templates/mysql.tmpl
@@ -2,6 +2,7 @@ resources:
   - type: sqladmin.v1beta4.instance
     name: '{{ .name }}'
     properties:
+      project: '{{ .project }}'
       region: '{{ .region }}'
       databaseVersion: '{{ .db_version }}'
       instanceType: CLOUD_SQL_INSTANCE
@@ -16,31 +17,36 @@ resources:
           requireSsl: true
         dataDiskSizeGb: 10
         dataDiskType: PD_SSD
-        activationPolicy: '{{ .activation_policy }}'
         locationPreference:
-          zone: {{ .zone }}
+          zone: '{{ .zone }}'
 
   - type: sqladmin.v1beta4.database
-    name: {{ .name }}-database
+    name: '{{ .name }}-database'
     properties:
-      name: {{ .database }}
+      project: '{{ .project }}'
+      name: '{{ .database }}'
       instance: $(ref.{{ .name }}.name)
       charset: utf8mb4
       collation: utf8mb4_general_ci
 
   - type: sqladmin.v1beta4.user
-    name: {{ .name }}-user
+    name: '{{ .name }}-user'
+    metadata:
+      dependsOn:
+        - '{{ .name }}'
+        - '{{ .name }}-database'
     properties:
-      name: {{ .username }}
-      password: {{ .password }}
+      project: '{{ .project }}'
+      name: '{{ .username }}'
+      password: '{{ .password }}'
       instance: $(ref.{{ .name }}.name)
 
 outputs:
   - name: EnvMysqlInstance
     value: $(ref.{{ .name }}.name)
   - name: EnvMysqlDatabase
-    value: {{ .database }}
+    value: '{{ .database }}'
   - name: EnvMysqlPassword
-    value: {{ .password }}
+    value: '{{ .password }}'
   - name: EnvMysqlUsername
-    value: {{ .username }}
+    value: '{{ .username }}'
diff --git a/cloud/google/templates/postgres.tmpl b/cloud/google/templates/postgres.tmpl
@@ -2,6 +2,7 @@ resources:
   - type: sqladmin.v1beta4.instance
     name: '{{ .name }}'
     properties:
+      project: '{{ .project }}'
       region: '{{ .region }}'
       databaseVersion: '{{ .db_version }}'
       instanceType: CLOUD_SQL_INSTANCE
@@ -10,27 +11,33 @@ resources:
         backupConfiguration:
           enabled: true
         locationPreference:
-          zone: {{ .zone }}
+          zone: '{{ .zone }}'
 
   - type: sqladmin.v1beta4.database
-    name: {{ .name }}-{{ .database }}
+    name: {{ .name }}-database
     properties:
+      project: '{{ .project }}'
       name: {{ .database }}
       instance: $(ref.{{ .name }}.name)
 
   - type: sqladmin.v1beta4.user
     name: {{ .name }}-user
+    metadata:
+      dependsOn:
+        - {{ .name }}
+        - {{ .name }}-database
     properties:
-      name: {{ .username }}
-      password: {{ .password }}
+      project: '{{ .project }}'
+      name: '{{ .username }}'
+      password: '{{ .password }}'
       instance: $(ref.{{ .name }}.name)
 
 outputs:
   - name: EnvPostgresInstance
     value: $(ref.{{ .name }}.name)
   - name: EnvPostgresDatabase
-    value: {{ .database }}
+    value: '{{ .database }}'
   - name: EnvPostgresPassword
-    value: {{ .password }}
+    value: '{{ .password }}'
   - name: EnvPostgresUsername
-    value: {{ .username }}
+    value: '{{ .username }}'
diff --git a/cmd/provider/aws/templates.go b/cmd/provider/aws/templates.go
diff --git a/cmd/provider/gcp/gcp.go b/cmd/provider/gcp/gcp.go
@@ -442,14 +442,6 @@ resources:
         - datacol
     metadata:
       items:
-      - key: DATACOL_API_KEY
-        value: {{ properties['api_key'] }}
-      - key: DATACOL_STACK
-        value: {{ properties['stack_name'] }}
-      - key: DATACOL_BUCKET
-        value: {{ properties['bucket'] }}
-      - key: DATACOL_CLUSTER
-        value: {{ properties['cluster_name'] }}
       - key: startup-script
         value: |
           #! /bin/bash
@@ -464,15 +456,18 @@ resources:
           DATACOL_STACK={{ properties['stack_name'] }}
           GCP_DEFAULT_ZONE={{ properties['zone'] }}
           GCP_REGION={{ properties['region'] }}
+          GCP_PROJECT={{ env['project'] }}
+          GCP_PROJECT_NUMBER={{ env['project_number'] }}
           EOF
+          while read line; do export $line; done < <(cat /etc/environment)
 
           curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.6.3/bin/linux/amd64/kubectl > kubectl &&
             chmod +x kubectl && \
             mv kubectl /usr/local/bin
+
           mkdir -p /opt/datacol && \
           curl -Ls /tmp https://storage.googleapis.com/{{ properties['artifact_bucket'] }}/binaries/{{ properties['version'] }}/apictl.zip > /tmp/apictl.zip
             unzip /tmp/apictl.zip -d /opt/datacol && chmod +x /opt/datacol/apictl
 
-          while read line; do export $line; done < <(cat /etc/environment)
           cd /opt/datacol && nohup ./apictl -log-file log.txt &
 `
diff --git a/cmd/stack.go b/cmd/stack.go
@@ -23,7 +23,7 @@ var (
 	credNotFound    = errors.New("Invalid credentials")
 	projectNotFound = errors.New("Invalid project id")
 
-	defaultGcpZone         = "asia-southeast1-a"
+	defaultGcpZone         = "asia-east1-a" //Taiwan
 	defaultAWSZone         = "ap-southeast-1a"
 	defaultAWSInstanceType = "t2.medium"
 	defaultGCPInstanceType = "n1-standard-1"