From b132e0010af28bc131f34222c5855fcd3e5b919a Mon Sep 17 00:00:00 2001 From: Jan Lauber Date: Sat, 5 Feb 2022 23:23:59 +0100 Subject: [PATCH 1/2] add quota support Signed-off-by: Jan Lauber --- .gitignore | 1 + README.md | 52 +++++++++++-------- controllers/quotaController.go | 92 ++++++++++++++++++++++++++++++++++ routes/routes.go | 29 +++-------- util/k8s.go | 24 +++++++++ util/os.go | 61 ++++++++++++++++++++++ util/quota.go | 8 +++ 7 files changed, 226 insertions(+), 41 deletions(-) create mode 100644 controllers/quotaController.go create mode 100644 util/quota.go diff --git a/.gitignore b/.gitignore index 6aa0f4b..42b7c10 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ docs/kubernetes/pod.yaml docs/kubernetes/pvc.yaml docs/kubernetes/oingress.yaml docs/kubernetes/ns.yaml +docs/kubernetes/ns-config.yaml .DS_Store \ No newline at end of file diff --git a/README.md b/README.md index a939975..8f84318 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,8 @@ Tenants represents the teams of a GitHub organization. #### `GET` > **important:** for authenticated access you need to provide the `Authorization` header with the `Bearer` token. -You can add `` in front of the path to get the tenant specific data (of everything). -> e.g. `/api/v1//pods` +You can add `` in front of the path to get the tenant specific data (of everything). +> e.g. `/api/v1//pods` #### auth `/login/github` - Login with GitHub \ `/login/github/callback` - Callback after GitHub login @@ -29,21 +29,27 @@ You can add `` in front of the path to get the tenant specific data `/api/v1/notifications` - Get the Slack notification messages of the broadcast channel provided via envs ##### general tenant resources -`/api/v1/pods` - Get pods of a tenant \ -`/api/v1/namespaces` - Get namespaces of a tenant \ -`/api/v1/serviceaccounts` - Get serviceaccounts of a tenant by namespaces \ +`/api/v1//pods` - Get pods of a tenant \ +`/api/v1//namespaces` - Get namespaces of a tenant \ +`/api/v1//serviceaccounts` - Get serviceaccounts of a tenant by namespaces \ ##### specific tenant resources -`/api/v1/requests/cpu` - Get cpurequests in **Milicores** of a tenant \ -`/api/v1/requests/memory` - Get memoryrequests in **Bytes** of a tenant \ -`/api/v1/requests/storage` - Get storagerequests in **Bytes** of a tenant by storageclass \ -`/api/v1/requests/ingress` - Get ingress ressources total of a tenant by ingressclass +`/api/v1//requests/cpu` - Get cpurequests in **Milicores** of a tenant \ +`/api/v1//requests/memory` - Get memoryrequests in **Bytes** of a tenant \ +`/api/v1//requests/storage` - Get storagerequests in **Bytes** of a tenant by storageclass \ +`/api/v1//requests/ingress` - Get ingress resourcess total of a tenant by ingressclass + +##### tenant resources costs +`/api/v1//costs/cpu` - Get the CPU costs by CPU \ +`/api/v1//costs/memory` - Get the memory costs by Memory \ +`/api/v1//costs/storage` - Get the storage costs by StorageClass \ +`/api/v1//costs/ingress` - Get the ingress costs by tenant + +##### tenant resource quotas +`/api/v1//quotas/cpu` - Get the CPU resource Quota by the label defined via env \ +`/api/v1//quotas/memory` - Get the memory resource Quota by the label defined via env \ +`/api/v1//quotas/storage` - Get the storage resource Quota for each storage class by the label**s** defined via env -##### tenant ressource costs -`/api/v1/costs/cpu` - Get the cpu costs by CPU \ -`/api/v1/costs/memory` - Get the memory costs by Memory \ -`/api/v1/costs/storage` - Get the storage costs by StorageClass \ -`/api/v1/costs/ingress` - Get the ingress costs by tenant #### `POST` @@ -67,19 +73,25 @@ You can send the github code with json body `{"github_code": "..."}` to the `/lo `SECRET_KEY` - JWT secret key *optional* (default: random 32 bytes, displayed in the logs) ### notifications -`SLACK_TOKEN` - Tenant API Slack Application User Token (if not set, the notification REST route will be deactivated) \ -`SLACK_BROADCAST_CHANNEL_ID` - BroadCast Slack Channel ID +`SLACK_TOKEN` - Tenant API Slack Application User Token *optional* (if not set, the notification REST route will be deactivated) \ +`SLACK_BROADCAST_CHANNEL_ID` - BroadCast Slack Channel ID *optional* (**required** if SLACK_TOKEN is set) -### tenant ressource identifiers -`TENANT_LABEL` - label key for selecting tenant ressources *optional* (default: "natron.io/tenant") +### tenant resources identifiers +`TENANT_LABEL` - label key for selecting tenant resourcess *optional* (default: "natron.io/tenant") ### cost calculation values `DISCOUNT_LABEL` - label key for selecting the discount value *optional* (default: "natron.io/discount" (float -> e.g. "0.1")) \ -`CPU_COST` - Cost of a cpu in your currency *optional* (default: 1.00 for 1 CPU) \ +`CPU_COST` - Cost of a CPU in your currency *optional* (default: 1.00 for 1 CPU) \ `MEMORY_COST` - Cost of a memory in your currency *optional* (default: 1.00 for 1 GB) \ -`STORAGE_COST_` - Cost of your storage classes in your currency *optional, multiple allowed* (default: 1.00 for 1 GB) \ +`STORAGE_COST_` - Cost of your storage classes in your currency **required, multiple allowed** (default: 1.00 for 1 GB) \ `INGRESS_COST` - Cost of ingress in your currency *optional* (default: 1.00 for 1 ingress) +### resource quotas +`QUOTA_NAMESPACE_SUFFIX` - The namespace suffix where the config of the tenant configuration takes place *optional* (default: "config" e.g. namespace name: "tenant-config") \ +`QUOTA_CPU_LABEL` - The CPU quota label *optional* (default: "natron.io/cpu-quota") \ +`QUOTA_MEMORY_LABEL` - The memory quota label *optional* (default: "natron.io/memory-quota") +`QUOTA_STORAGE_LABEl_` - The storage label of each storage class *optional, multiple allowed* (default: "natron.io/storage-quota-" renders every storageclass defined in the Storage) + ## deployment *example deployment files:* [kubernetes manifests](docs/kubernetes) diff --git a/controllers/quotaController.go b/controllers/quotaController.go new file mode 100644 index 0000000..db04cda --- /dev/null +++ b/controllers/quotaController.go @@ -0,0 +1,92 @@ +package controllers + +import ( + "github.com/gofiber/fiber/v2" + "github.com/natron-io/tenant-api/util" +) + +// GetCPUQuota returns the CPU quota of a tenant by the label at the tenant config namespace +func GetCPUQuota(c *fiber.Ctx) error { + + util.InfoLogger.Printf("%s %s %s", c.IP(), c.Method(), c.Path()) + tenant := c.Params("tenant") + tenants := CheckAuth(c) + if len(tenants) == 0 { + return c.Status(401).JSON(fiber.Map{ + "message": "Unauthorized", + }) + } + if tenant != "" && !util.Contains(tenant, tenants) { + return c.Status(403).JSON(fiber.Map{ + "message": "Forbidden", + }) + } + + cpuQuota, err := util.GetRessourceQuota(tenant, util.QUOTA_NAMESPACE_SUFFIX, util.QUOTA_CPU_LABEL) + if err != nil { + return c.Status(500).JSON(fiber.Map{ + "message": "Internal Server Error", + }) + } + + return c.JSON(cpuQuota) +} + +// GetMemoryQuota returns the Memory quota of a tenant by the label at the tenant config namespace +func GetMemoryQuota(c *fiber.Ctx) error { + + util.InfoLogger.Printf("%s %s %s", c.IP(), c.Method(), c.Path()) + tenant := c.Params("tenant") + tenants := CheckAuth(c) + if len(tenants) == 0 { + return c.Status(401).JSON(fiber.Map{ + "message": "Unauthorized", + }) + } + if tenant != "" && !util.Contains(tenant, tenants) { + return c.Status(403).JSON(fiber.Map{ + "message": "Forbidden", + }) + } + + memoryQuota, err := util.GetRessourceQuota(tenant, util.QUOTA_NAMESPACE_SUFFIX, util.QUOTA_MEMORY_LABEL) + if err != nil { + return c.Status(500).JSON(fiber.Map{ + "message": "Internal Server Error", + }) + } + + return c.JSON(memoryQuota) +} + +// GetStorageQuota returns the Storage quota of a tenant by the label at the tenant config namespace +func GetStorageQuota(c *fiber.Ctx) error { + + util.InfoLogger.Printf("%s %s %s", c.IP(), c.Method(), c.Path()) + tenant := c.Params("tenant") + tenants := CheckAuth(c) + if len(tenants) == 0 { + return c.Status(401).JSON(fiber.Map{ + "message": "Unauthorized", + }) + } + if tenant != "" && !util.Contains(tenant, tenants) { + return c.Status(403).JSON(fiber.Map{ + "message": "Forbidden", + }) + } + + storageQuota := make(map[string]float64) + var err error + + for key, value := range util.QUOTA_STORAGE_LABEL { + storageQuota[key], err = util.GetRessourceQuota(tenant, util.QUOTA_NAMESPACE_SUFFIX, value) + if err != nil { + return c.Status(500).JSON(fiber.Map{ + "message": "Internal Server Error", + }) + } + } + + return c.JSON(storageQuota) +} diff --git a/routes/routes.go b/routes/routes.go index 8b94813..3717ed7 100644 --- a/routes/routes.go +++ b/routes/routes.go @@ -26,41 +26,28 @@ func Setup(app *fiber.App, clientset *kubernetes.Clientset) { // Tenants v1.Get("/tenants", controllers.GetTenants) - // Every Tenant - v1.Get("/pods", controllers.GetPods) - v1.Get("/namespaces", controllers.GetNamespaces) - v1.Get("/serviceAccounts", controllers.GetServiceAccounts) - // Specific Tenant v1.Get(":tenant/pods", controllers.GetPods) v1.Get(":tenant/namespaces", controllers.GetNamespaces) v1.Get(":tenant/serviceAccounts", controllers.GetServiceAccounts) - // Every Tenant - requests := v1.Group("/requests") - requests.Get("/cpu", controllers.GetCPURequestsSum) - requests.Get("/memory", controllers.GetMemoryRequestsSum) - requests.Get("/storage", controllers.GetStorageRequestsSum) - requests.Get("/ingress", controllers.GetIngressRequestsSum) - // Specific Tenant - requests = v1.Group(":tenant/requests") + requests := v1.Group(":tenant/requests") requests.Get("/cpu", controllers.GetCPURequestsSum) requests.Get("/memory", controllers.GetMemoryRequestsSum) requests.Get("/storage", controllers.GetStorageRequestsSum) requests.Get("/ingress", controllers.GetIngressRequestsSum) - // Every Tenant - costs := v1.Group("/costs") - costs.Get("/cpu", controllers.GetCPUCostSum) - costs.Get("/memory", controllers.GetMemoryCostSum) - costs.Get("/storage", controllers.GetStorageCostSum) - costs.Get("/ingress", controllers.GetIngressCostSum) - // Per tenant - costs = v1.Group(":tenant/costs") + costs := v1.Group(":tenant/costs") costs.Get("/cpu", controllers.GetCPUCostSum) costs.Get("/memory", controllers.GetMemoryCostSum) costs.Get("/storage", controllers.GetStorageCostSum) costs.Get("/ingress", controllers.GetIngressCostSum) + + // Quotas + quotas := v1.Group(":tenant/quotas") + quotas.Get("/cpu", controllers.GetCPUQuota) + quotas.Get("/memory", controllers.GetMemoryQuota) + quotas.Get("/storage", controllers.GetStorageQuota) } diff --git a/util/k8s.go b/util/k8s.go index df63161..207c42f 100644 --- a/util/k8s.go +++ b/util/k8s.go @@ -287,3 +287,27 @@ func GetIngressRequestsSumByTenant(tenants []string) (map[string][]string, error return tenantsIngress, nil } + +// GetRessourceQuota returns the resource quota for the given tenant and label set in the config namespace +func GetRessourceQuota(tenant string, namespace_suffix string, label string) (float64, error) { + // get the namespace with tenant-namespace_suffix and get the label value + namespace, err := Clientset.CoreV1().Namespaces().Get(context.TODO(), tenant+"-"+namespace_suffix, metav1.GetOptions{}) + if err != nil { + return 0, err + } + + // get the cpu quota from the label + cpuQuota := namespace.Labels[label] + if cpuQuota == "" { + cpuQuota = "0" + } + + // convert to float64 + cpuQuotaFloat, err := strconv.ParseFloat(cpuQuota, 64) + if err != nil || cpuQuotaFloat < 0 { + WarningLogger.Printf("CPU quota value %s is not valid for pod %s with label %s", cpuQuota, namespace.Name, label) + cpuQuota = "0" + } + + return cpuQuotaFloat, nil +} diff --git a/util/os.go b/util/os.go index a9704ae..7018e6d 100644 --- a/util/os.go +++ b/util/os.go @@ -94,6 +94,7 @@ func LoadEnv() error { } // get every env variable starting with STORAGE_COST_ and parse it to STORAGE_COST with the storage class name after STORAGE_COST_ as key + storageClasses := []string{} tempStorageCost := make(map[string]map[string]float64) for _, env := range os.Environ() { if strings.HasPrefix(env, "STORAGE_COST_") { @@ -112,6 +113,7 @@ func LoadEnv() error { // add to tempStorageCost tempStorageCost[key[2]] = map[string]float64{"cost": value} InfoLogger.Printf("storage class %s set to cost value: %f", key[2], value) + storageClasses = append(storageClasses, key[2]) } } STORAGE_COST = tempStorageCost @@ -132,5 +134,64 @@ func LoadEnv() error { InfoLogger.Printf("INGRESS_COST set to: %f", INGRESS_COST) } + if QUOTA_CPU_LABEL = os.Getenv("QUOTA_CPU_LABEL"); QUOTA_CPU_LABEL == "" { + WarningLogger.Println("QUOTA_CPU_LABEL is not set") + QUOTA_CPU_LABEL = "natron.io/cpu-quota" + InfoLogger.Printf("QUOTA_CPU_LABEL set using default: %s", QUOTA_CPU_LABEL) + } else { + InfoLogger.Printf("QUOTA_CPU_LABEL set using env: %s", QUOTA_CPU_LABEL) + } + + if QUOTA_MEMORY_LABEL = os.Getenv("QUOTA_MEMORY_LABEL"); QUOTA_MEMORY_LABEL == "" { + WarningLogger.Println("QUOTA_MEMORY_LABEL is not set") + QUOTA_MEMORY_LABEL = "natron.io/memory-quota" + InfoLogger.Printf("QUOTA_MEMORY_LABEL set using default: %s", QUOTA_MEMORY_LABEL) + } else { + InfoLogger.Printf("QUOTA_MEMORY_LABEL set using env: %s", QUOTA_MEMORY_LABEL) + } + + if QUOTA_NAMESPACE_SUFFIX = os.Getenv("QUOTA_NAMESPACE_SUFFIX"); QUOTA_NAMESPACE_SUFFIX == "" { + WarningLogger.Println("QUOTA_NAMESPACE_SUFFIX is not set") + QUOTA_NAMESPACE_SUFFIX = "config" + InfoLogger.Printf("QUOTA_NAMESPACE_SUFFIX set using default: %s", QUOTA_NAMESPACE_SUFFIX) + } else { + InfoLogger.Printf("QUOTA_NAMESPACE_SUFFIX set using env: %s", QUOTA_NAMESPACE_SUFFIX) + } + + // for each storageclass get the quota label + QUOTA_STORAGE_LABEL = make(map[string]string) + // get storage class names from env + for _, env := range os.Environ() { + if strings.HasPrefix(env, "QUOTA_STORAGE_LABEL_") { + // split env variable to key and value + keyValue := strings.Split(env, "=") + // split key to storage class name and cost + key := strings.Split(keyValue[0], "_") + // add to tempStorageCost + + // check if storage class already exists in storageClasses + for _, storageClass := range storageClasses { + if storageClass == key[3] { + break + } + } + storageClasses = append(storageClasses, key[3]) + } + } + + for _, storageClass := range storageClasses { + label := os.Getenv("QUOTA_STORAGE_LABEL_" + storageClass) + if label == "" { + WarningLogger.Printf("QUOTA_STORAGE_LABEL_%s is not set", storageClass) + label = "natron.io/storage-quota-" + storageClass + InfoLogger.Printf("QUOTA_STORAGE_LABEL_%s set using default: %s", storageClass, label) + // add to QUOTA_STORAGE_LABEL + QUOTA_STORAGE_LABEL[storageClass] = label + } else { + InfoLogger.Printf("QUOTA_STORAGE_LABEL_%s set using env: %s", storageClass, label) + QUOTA_STORAGE_LABEL[storageClass] = label + } + } + return nil } diff --git a/util/quota.go b/util/quota.go new file mode 100644 index 0000000..abda7d1 --- /dev/null +++ b/util/quota.go @@ -0,0 +1,8 @@ +package util + +var ( + QUOTA_NAMESPACE_SUFFIX string + QUOTA_CPU_LABEL string + QUOTA_MEMORY_LABEL string + QUOTA_STORAGE_LABEL map[string]string +) From 75cbd39554e20a43a5bba559bf9ed32fd7553c88 Mon Sep 17 00:00:00 2001 From: Jan Lauber Date: Sat, 5 Feb 2022 23:27:02 +0100 Subject: [PATCH 2/2] increase jwt timeout to one day Signed-off-by: Jan Lauber --- controllers/authController.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/controllers/authController.go b/controllers/authController.go index 34a5d11..f332704 100644 --- a/controllers/authController.go +++ b/controllers/authController.go @@ -91,8 +91,8 @@ func LoggedIn(c *fiber.Ctx, githubData string) error { }) } - // expire token in 1 hour - exp := time.Now().Add(time.Hour).Unix() + // expire token in 1 day + exp := time.Now().Add(time.Hour * 24).Unix() claims := jwt.MapClaims{ "github_team_slugs": githubTeamSlugs,