From 48916932188be1512b5d7ab4f689ce1464f329c3 Mon Sep 17 00:00:00 2001 From: Karsten Gorskowski Date: Mon, 4 Dec 2023 15:30:36 +0100 Subject: [PATCH] rewrite --- helm/mapping-values.yaml | 128 +++++++++++ helm/template-values.yaml | 316 ++++++++++++++++++++++++++ tenant/commands/init.py | 33 +-- tenant/commands/resize.py | 15 +- tenant/utils/common.py | 1 + tenant/utils/generate_secrets_file.py | 14 ++ tenant/utils/template_values.py | 104 +++++++++ 7 files changed, 597 insertions(+), 14 deletions(-) create mode 100644 helm/mapping-values.yaml create mode 100644 helm/template-values.yaml create mode 100644 tenant/utils/template_values.py diff --git a/helm/mapping-values.yaml b/helm/mapping-values.yaml new file mode 100644 index 0000000..2aa5564 --- /dev/null +++ b/helm/mapping-values.yaml @@ -0,0 +1,128 @@ +defaults: + elasticsearch: + version: 8.11.1 + kibana: + version: 8.11.1 +S: + elasticsearch: + coordinator: + dedicated: false + replicas: 0 + resources: + limits: + cpu: 1 + memory: 2Gi + requests: + cpu: 1 + memory: 2Gi + storage: 1Gi + storageClass: topolvm-provisioner + data: + replicas: 3 + resources: + limits: + cpu: 1 + memory: 4Gi + requests: + cpu: 1 + memory: 4Gi + storage: 30Gi + storageClass: topolvm-provisioner + master: + dedicated: true + replicas: 3 + resources: + limits: + cpu: 500m + memory: 2Gi + requests: + cpu: 500m + memory: 2Gi + storage: 1Gi + storageClass: topolvm-provisioner + kafka: + replicas: 3 + resources: + limits: + cpu: 750m + memory: 2Gi + requests: + cpu: 750m + memory: 2Gi + storage: 30Gi + storageClass: topolvm-provisioner + topics: + logging: + partitions: 3 + replication: 2 + retentionBytes: "1000000000" + retentionMs: "86400000" + segmentBytes: "250000000" + segmentMs: "3600000" + pipelines: + replicas: 2 + logging: + name: simple + workers: 3 + +M: + elasticsearch: + coordinator: + dedicated: true + replicas: 3 + resources: + limits: + cpu: 1 + memory: 2Gi + requests: + cpu: 1 + memory: 2Gi + storage: 1Gi + storageClass: topolvm-provisioner + data: + replicas: 5 + resources: + limits: + cpu: 2 + memory: 4Gi + requests: + cpu: 2 + memory: 4Gi + storage: 50Gi + storageClass: topolvm-provisioner + master: + dedicated: true + replicas: 3 + resources: + limits: + cpu: 1 + memory: 4Gi + requests: + cpu: 1 + memory: 4Gi + storage: 1Gi + storageClass: topolvm-provisioner + kafka: + replicas: 3 + resources: + limits: + cpu: 1 + memory: 4Gi + requests: + cpu: 1 + memory: 4Gi + storage: 50Gi + storageClass: topolvm-provisioner + topics: + logging: + partitions: 3 + replication: 2 + retentionBytes: "1000000000" + retentionMs: "86400000" + segmentBytes: "250000000" + segmentMs: "3600000" + pipelines: + replicas: 3 + logging: + name: simple + workers: 3 \ No newline at end of file diff --git a/helm/template-values.yaml b/helm/template-values.yaml new file mode 100644 index 0000000..bd79baa --- /dev/null +++ b/helm/template-values.yaml @@ -0,0 +1,316 @@ +# set standard values for new elastic tenant helm values +--- +elasticsearch: + config: + flavor: placeholder + logging: + enabled: false + rbac: + customUsers: + logstash_writer: + email: + enabled: true + roles: + - logstash_admin_user + - logstash_internal + userRoles: + elastic-superuser: + applications: + - application: "*" + privileges: + - "*" + resources: + - "*" + cluster: + - all + indices: + - allow_restricted_indices: true + names: "*" + privileges: + - all + run_as: + - "*" + logstash_writer: + applications: [ ] + cluster: + - manage_index_templates + - monitor + - manage_ilm + indices: + - field_security: + grant: + - "*" + names: + - pn-* + - pits-* + - test-* + - dead-letter-* + - customer-* + privileges: + - write + - create + - create_index + - manage + - manage_ilm + metadata: { } + run_as: [ ] + transient_metadata: + enabled: true + prometheus_reader: + applications: [ ] + cluster: + - cluster:monitor/prometheus/metrics + - cluster:monitor/health + - cluster:monitor/nodes/stats + - cluster:monitor/state + indices: + - names: "*" + privileges: + - monitor + xpack: + license: + self_generated_type: basic + security: + accessAgreement: | + Sie sind dabei, auf potentiell durch die DSGVO geschützte Daten + zuzugreifen. + + Durch Einloggen in dieses System bestätigen Sie, Kenntnis genommen + zu haben, dass ihre Nutzung zwecks Auditierung protokolliert wird. + Sie sind dem Datengeheimnis iSd Art. 29 DSGVO verpflichtet. + Unautorisierte Nutzung des Systems ist strikt untersagt und kann + rechtliche Konsequenzen zur Folge haben. + + Vor einer dienstlich notwendigen Übermittlung von Log-Daten mit + personenbezogenen Daten ist eine (Pseudo-)Anonymisierung + durchzuführen. Hierzu ist eine Datenschutzfolgeabschätzung beim + Datenschutzbeauftragten ITZ vorzulegen. + + Für nicht anonymisierte Daten ist ein solcher Versand nur zulässig, + wenn mit dem Hersteller eine Auftragsdatenvereinbarung nach Art. 28 + DSGVO oder § 62 BDSG, bei Auslandsbezug gem. Art. 44 - 46 DSGVO + und/oder § 79 - 83 BDSG, abgeschlossen wurde. Zuwiderhandlung stellt + regelmäßig einen Datenschutzverletzung dar. + audit: + enabled: false + + ilmPolicies: + logging-std-7d: + policy: + phases: + delete: + actions: + delete: + delete_searchable_snapshot: true + min_age: 7d + hot: + actions: { } + min_age: 0ms + image: + version: placeholder + indexTemplates: + logging: + index_patterns: + - logging-* + template: + settings: + index: + lifecycle: + name: logging-std-7d + number_of_replicas: 1 + number_of_shards: 5 + unassigned: + node_left: + delayed_timeout: 15m + ml: + dedicated: false +epr: + enabled: false +ingress: + elasticsearch: + enabled: false + inspect: false + epr: + enabled: false + kibana: + enabled: true + inspect: false + kowl: + enabled: false + logstash: + inspect: false +kafka: + config: + logging: + enabled: false + javaOpts: + heap: -Xmx1g -Xms1g + replicas: 3 + # TODO: should be removed and replaced with "normal" kafka img when all tenants run `self-managed/kafka` + setup: + image: + name: laas-craas.bcsv.cloud.itzbund.net/laas/self-managed/kafka + version: 7.4.0-1 + + topics: + # TODO: 1) retention time of 7d is sufficient for §100/§5 tenants + # TODO: 2) decide whether topics should be created here or in Logstash + logging: + partitions: 3 + replication: 2 + # NOTICE: max capacity: repl * part * rBytes + influx(gcFreq = 5min) + retentionBytes: "1400000000" + retentionMs: "86400000" + segmentBytes: "500000000" + segmentMs: "3600000" + zookeeper: + config: + logging: + enabled: false + javaOpts: + heapSize: 500m + replicas: 3 + resources: + limits: + cpu: 300m + memory: 1000Mi + requests: + cpu: 300m + memory: 1000Mi + storage: 1Gi + storageClass: topolvm-provisioner +kibana: + apiVersion: post710 + config: + logging: + enabled: true + monitoring: + enabled: true + rbac: + userRoles: + kibana-user: + elasticsearch: + cluster: [ ] + indices: + - allow_restricted_indices: false + names: + - logging-* + privileges: + - read + - monitor + - view_index_metadata + run_as: [ ] + kibana: + - base: [ ] + feature: + dashboard: + - all + discover: + - all + savedObjectsManagement: + - all + savedObjectsTagging: + - all + visualize: + - all + xpack: + security: + cookieName: sid + enabled: true + image: + version: placeholder + replicas: 1 + resources: + limits: + cpu: 500m + memory: 2Gi + requests: + cpu: 500m + memory: 2Gi +logstash: + config: + logging: + enabled: false + gitSync: + # TODO: use either branch name for floating versions, + # TODO: tag/commit as fixed version equivalents + branch: itzb/release + username: tenant-reader + image: + version: 8.7.1 + ingest: + codec: json + enabled: true + hostNetwork: false + inProto: http + javaOpts: + heap: -Xms2000m -Xmx2000m + nodePort: null + nodeSelectors: null + port: 8443 + replicas: 2 + resources: + limits: + cpu: 2 + memory: 4Gi + requests: + cpu: 2 + memory: 4Gi + pipeline: + config: + deadLetterQueue: false + enabled: true + javaOpts: + heap: -Xms2000m -Xmx2000m + memcached: + enabled: false + pipelines: + - customer: laas + id: simple + injections: + inputThreads: 3 + name: simple + workers: 3 + pseudonymize: false + replicas: 3 + resources: + limits: + cpu: 2 + memory: 4Gi + requests: + cpu: 2 + memory: 4Gi + syncPipelines: false +oauthProxy: + allowedGroups: [ ] + emailClaim: email + groupsClaim: groups + image: + name: laas-craas.bcsv.cloud.itzbund.net/laas/self-managed/oauth2-proxy + version: v7.4.0 + issuerUrl: + host: idp.laas.cloud.itz.in.bund.de + request_logging: true + scope: + - email + - groups + - openid + - profile +tls: + externalCertificates: + kibana: + tls_crt: | + -----BEGIN CERTIFICATE----- + # Insert Certificate here when obtained from DTRust + -----END CERTIFICATE----- + tls_key: placeholderOverwriteFromSecret + issuer: + auth: + path: placeholder + role: placeholder + name: placeholder + secret: + role: placeholder + serviceAccount: default + organizations: + - ITZB diff --git a/tenant/commands/init.py b/tenant/commands/init.py index d36c977..966a2a1 100644 --- a/tenant/commands/init.py +++ b/tenant/commands/init.py @@ -2,8 +2,8 @@ import os from tenant.utils.common import get_secure_password, generate_key, generate_csr from tenant.utils.terraform import create_tfvars_file -import tenant.utils.generatecsr import tenant.utils.generate_secrets_file +from tenant.utils.template_values import template_values def add_subparser(subparsers): @@ -71,18 +71,18 @@ def execute(args): source_tf_dir = os.path.join(target_directory, "terraform") target_tf_dir = terraform_directory - for filename in os.listdir(source_tf_dir): - if filename.endswith(".tf"): - source_path = os.path.join(source_tf_dir, filename) - target_path = os.path.join(target_tf_dir, filename) - # Ensure the source path is correct before creating the symbolic link - if os.path.exists(source_path): - relative_path = os.path.relpath(source_path, target_tf_dir) - os.symlink(relative_path, target_path) - else: - print( - f"Warning: Source file '{filename}' not found in '{source_tf_dir}'." - ) + # for filename in os.listdir(source_tf_dir): + # if filename.endswith(".tf"): + # source_path = os.path.join(source_tf_dir, filename) + # target_path = os.path.join(target_tf_dir, filename) + # # Ensure the source path is correct before creating the symbolic link + # if os.path.exists(source_path): + # relative_path = os.path.relpath(source_path, target_tf_dir) + # os.symlink(relative_path, target_path) + # else: + # print( + # f"Warning: Source file '{filename}' not found in '{source_tf_dir}'." + # ) variables = { "tenant_name": tenant_name, @@ -91,6 +91,13 @@ def execute(args): tfvars_filepath = os.path.join(terraform_directory, tenant_name + ".tfvars") create_tfvars_file(variables, tfvars_filepath) + values_file = os.path.join(helm_directory, tenant_name + ".values.yaml") + if os.path.exists(values_file): + print("Values file already exists") + print(values_file) + else: + template_values(tenant_name, "S", "laas", ingress, values_file) + # generate secrets file if not already exist, not yet encrypted on the fly secrets_file = os.path.join(helm_directory, tenant_name + ".secrets.yaml") if os.path.exists(secrets_file): diff --git a/tenant/commands/resize.py b/tenant/commands/resize.py index a87f351..702a2f4 100644 --- a/tenant/commands/resize.py +++ b/tenant/commands/resize.py @@ -1,18 +1,31 @@ # tenant/commands/resize.py from tenant.utils.common import get_secure_password +import os +from tenant.utils.template_values import template_values def add_subparser(subparsers): resize_parser = subparsers.add_parser( "resize", help="Resize resources for a tenant" ) resize_parser.add_argument("tenant_name", help="Name of the tenant") - resize_parser.add_argument("--new-size", type=int, help="New size for resources") + resize_parser.add_argument("--new-size", help="New size for resources") + resize_parser.add_argument( + "--target", default=".", help="Target directory (default: current directory)" + ) def execute(args): + ingress = input( + "Please enter the FQDN of the Kibana ingress, without the 'kibana' prefix: " + ) + target_directory = args.target tenant_name = args.tenant_name new_size = args.new_size + tenant_directory = os.path.join(target_directory, tenant_name) + helm_directory = os.path.join(tenant_directory, "03-helm") + values_file = os.path.join(helm_directory, tenant_name + ".values.yaml") + template_values(tenant_name, new_size, "laas", ingress, values_file) # Use the shared function # shared_function() diff --git a/tenant/utils/common.py b/tenant/utils/common.py index 220785a..64e151d 100644 --- a/tenant/utils/common.py +++ b/tenant/utils/common.py @@ -1,5 +1,6 @@ # tenant/utils/common.py import getpass +import re from OpenSSL import crypto TYPE_RSA = crypto.TYPE_RSA diff --git a/tenant/utils/generate_secrets_file.py b/tenant/utils/generate_secrets_file.py index 348134b..5a77476 100644 --- a/tenant/utils/generate_secrets_file.py +++ b/tenant/utils/generate_secrets_file.py @@ -2,6 +2,7 @@ import random import os import ruamel.yaml import string +import subprocess yaml = ruamel.yaml.YAML() @@ -65,3 +66,16 @@ def generate_secrets_file(secrets_file, git_sync_password): }, file, ) + try: + subprocess.run( + [ + "sops", + "-e", + "--in-place", + "--hc-vault-transit=https://vault.laas.cloud.itz.in.bund.de/v1/sops/key/laas-bn-infra", + secrets_file, + ], + check=True, + ) + except subprocess.CalledProcessError as e: + print(f"Error: {e.returncode}\n{e.stderr}") diff --git a/tenant/utils/template_values.py b/tenant/utils/template_values.py new file mode 100644 index 0000000..33cf561 --- /dev/null +++ b/tenant/utils/template_values.py @@ -0,0 +1,104 @@ +import ruamel.yaml +import os +from tenant.utils.common import calculate_java_settings +from OpenSSL import crypto + +TYPE_RSA = crypto.TYPE_RSA +TYPE_DSA = crypto.TYPE_DSA +key = crypto.PKey() + +yaml = ruamel.yaml.YAML() + + +def recursively_sort_dict(input_dict): + sorted_dict = dict(sorted(input_dict.items())) + for key, value in sorted_dict.items(): + if isinstance(value, dict): + sorted_dict[key] = recursively_sort_dict(value) + return sorted_dict + + +def template_values(tenant_name, tenant_size, flavor, ingress, values_file): + print(os.getcwd()) + with open("./helm/template-values.yaml", "r", encoding="utf-8") as file: + existing_values = yaml.load(file) + + with open("./helm/mapping-values.yaml", "r") as file: + mapping = yaml.load(file) + + if tenant_size in mapping: + if flavor not in ["laas", "daas"]: + print("Invalid flavor") + else: + # map "selected_values" to "mapping" in reference to the tenant size + mapped_values = mapping[tenant_size] + + # setting default values from defaults block + existing_values["oauthProxy"]["clientId"] = tenant_name + "-logging-client" + existing_values["oauthProxy"]["issuerUrl"]["realmPath"] = ( + "/realms/" + tenant_name + ) + existing_values["elasticsearch"]["image"]["version"] = mapping["defaults"][ + "elasticsearch" + ]["version"] + + # configure resource sizing of elasticsearch components + existing_values["elasticsearch"]["config"]["flavor"] = flavor + + existing_values["elasticsearch"]["data"] = mapped_values["elasticsearch"][ + "data" + ] + existing_values["elasticsearch"]["coordinator"] = mapped_values[ + "elasticsearch" + ]["coordinator"] + existing_values["elasticsearch"]["master"] = mapped_values["elasticsearch"][ + "master" + ] + # configure kibana + existing_values["kibana"]["image"]["version"] = mapping["defaults"][ + "kibana" + ]["version"] + + # configure resource sizing of kafka components + existing_values["kafka"]["resources"] = mapped_values["kafka"]["resources"] + existing_values["kafka"]["javaOpts"]["heap"] = calculate_java_settings( + mapped_values["kafka"]["resources"]["requests"]["memory"] + ) + + # explicitly set ingress domain + existing_values["ingress"]["domain"] = ingress + + # template tls configuration + existing_values["tls"]["issuer"]["name"] = tenant_name + "-issuer" + existing_values["tls"]["issuer"]["auth"]["path"] = ( + "/v1/auth/bn-" + tenant_name + "-cert-manager" + ) + existing_values["tls"]["issuer"]["auth"]["role"] = ( + "bn-" + tenant_name + "-pki-INT-cert-signers" + ) + existing_values["tls"]["issuer"]["secret"]["role"] = ( + "bn-" + tenant_name + "-pki-INT-cert-signers" + ) + + existing_values["logstash"]["pipeline"]["pipelines"][0][ + "workers" + ] = mapped_values["pipelines"]["logging"]["workers"] + + existing_values["logstash"]["pipeline"]["replicas"] = mapped_values[ + "pipelines" + ]["replicas"] + + existing_values["kafka"]["topics"]["logging"]["partitions"] = ( + mapped_values["pipelines"]["replicas"] + * mapped_values["pipelines"]["logging"]["workers"] + ) + + # order the values in the "existing_values" dictionary alphabetically + existing_values = recursively_sort_dict(existing_values) + + # write ordered values to instance-new.yaml + with open(values_file, "w", encoding="utf-8") as new_file: + yaml.dump(existing_values, new_file) + + else: + print("Invalid tenant sizing")