Compare revisions

schnarkus · schnarkus · schnarkus · schnarkus · schnarkus · schnarkus
--- a/.gitignore
+++ b/.gitignore
 .local
+artifact.bin
+*terraform*
+.ssh
+kubectl*
+db_pass
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
+---
 workflow:
  rules:
-    - if: >-
-        $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main"
-      when: 'always'
-    - when: 'never'
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event" || ($CI_PIPELINE_SOURCE == "push" && ($CI_COMMIT_BRANCH == "main" || $CI_COMMIT_BRANCH == "prod" || $CI_COMMIT_BRANCH == "dev"))
+      when: always
+    - when: never

+variables:
+  version: 0.0.$CI_PIPELINE_IID

-job_trigger-pipeline:
-  trigger:
-    project: 'fb6-wp11-devops/webservice-build-and-publish'
+stages:
+  - test
+  - build
+  - publish
+
+test_job:
+  stage: test
+  image: public.ecr.aws/docker/library/golang:1.21
+  script:
+    - go get -t ./...
+    - go test -race -v ./...
+
+build_job:
+  stage: build
+  rules:
+    - if: $CI_COMMIT_REF_NAME =~ /prod|dev/
+      when: always
+  image: public.ecr.aws/docker/library/golang:1.21
+  script: |
+    echo "$DB_PASSWORD" > db_pass
+    GOARCH=amd64 GOOS=linux CGO_ENABLED=0 go build -o artifact.bin ./*.go
+  artifacts:
+    paths:
+      - artifact.bin
+      - db_pass
+    expire_in: 5 min
+
+publish_job:
+  stage: publish
+  rules:
+    - if: $CI_COMMIT_REF_NAME =~ /prod|dev/
+      when: always
+  image: docker:latest
+  services:
+    - docker:dind
+  tags:
+    - docker-privileged
+  dependencies:
+    - build_job
+  script:
+    - docker build -t $CI_REGISTRY_IMAGE:$version -f Containerfile .
+    - docker tag $CI_REGISTRY_IMAGE:$version $CI_REGISTRY_IMAGE:latest
+    - if [[ "$CI_COMMIT_BRANCH" == "prod" ]]; then
+      docker tag $CI_REGISTRY_IMAGE:$version $CI_REGISTRY_IMAGE:prod;
+      elif [[ "$CI_COMMIT_BRANCH" == "dev" ]]; then
+      docker tag $CI_REGISTRY_IMAGE:$version $CI_REGISTRY_IMAGE:dev;
+      fi
+    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
+    - docker push $CI_REGISTRY_IMAGE:$version
+    - docker push $CI_REGISTRY_IMAGE:latest
+    - if [[ "$CI_COMMIT_BRANCH" == "prod" ]]; then
+      docker push $CI_REGISTRY_IMAGE:prod;
+      elif [[ "$CI_COMMIT_BRANCH" == "dev" ]]; then
+      docker push $CI_REGISTRY_IMAGE:dev;
+      fi
+  after_script:
+    - docker logout $CI_REGISTRY
--- a/Containerfile
+++ b/Containerfile
+# syntax=docker/dockerfile:1
+
+FROM ubuntu:22.04
+
+RUN apt-get update && apt-get install -y redis-server
+
+COPY artifact.bin /artifact.bin
+COPY db_pass /db_pass
+
+# Set more environment variables for the web service
+ENV HOST=0.0.0.0
+ENV PORT=8080
+
+ENV DB_HOST=0.0.0.0
+ENV DB_PORT=6379
+ENV DB_NAME=0
+ENV DB_PASSWORD=/db_pass
+
+ENV FONT_COLOR=blue
+
+EXPOSE 8080
+
+CMD redis-server --daemonize yes && /artifact.bin
\ No newline at end of file
--- a/Lifecycle.png
+++ b/Lifecycle.png
--- a/README.md
+++ b/README.md
-Webservice
-==========
+# Deliverable: Projektkonzept

-A Go-based simple web service meant to be the subject of any tutorial
-or even used the project work.
+## Technologiewahl

+- **Go**:
+  Vorgegebene Programmiersprache für den Webservice.
+- **GitLab CI/CD**:
+  Das Hochschul-GitLab bietet die Infrastruktur für die CI/CD-Pipeline, inklusive Container-Registry und Quellcode-Verwaltung.
+- **Podman**:
+  Container-Runtime zur Verwaltung, ähnlich wie Docker, jedoch auch ohne Administratorrechte benutzbar.
+- **Kubernetes**:
+  Plattform für Cluster- und Pod-Management, Schnittstelle zur Google Cloud sowie zahlreiche Dienste wie Redis und integrierte Dashboards zur Überwachung der Systeme.
+- **OpenTofu**:
+  Management-Tool für deklarative Infrastrukturverwaltung, ermöglicht idempotenten Aufbau der Zielstruktur.

-#### Prerequisites:
+## Umgebungen

-* minimal [required version](./go.mod#L3) of the Go toolchain (install via
-  system package manager or [by hand](https://go.dev/doc/install))
-* [optional] [Redis](https://redis.io/docs/install/) to persist state
+- **Entwicklung**: Lokal mit Podman-Containern
+- **Produktion**: Global über Kubernetes-Cluster in GCP

+## Dienste <a id="Dienste"></a>

-#### State:
+- **Compute Engine**: Hostet VMs für Webservice und Redis-Datenbank
+- **Kubernetes Engine**: Autopilot-Cluster für Webservice und Ingress-Controller

-If the database host is not explicitly defined, then the state is ephemeral. For more
-information checkout the [configuration code](./configuration/config.go).
+## Infrastruktur

+Die Produktivumgebung des Webservice wird auf der Google Cloud Platform (GCP) gehostet:

-#### Build:
+- **Details zur Infrastruktur**:

-1. Install dependencies: `go get -t ./...`
-2. Run locally: `go run .`
-3. Execute unit tests: `go test -race -v ./...`
-4. Build artifact: `go build -o ./artifact.bin ./*.go`
+  - **Maschinentyp**: e2-medium (2 vCPUs, 4 GB RAM)
+  - **Region**: Frankfurt
+  - **Betriebssystem**: Ubuntu 22.04
+  - **Festplatte**: 20 GB

-To build for another platform, set `GOOS` and `GOARCH`. To yield a static
-binary (fully self-contained, no dynamic linking) set `CGO_ENABLED=0`. 
-To set a version during build time, add the following CLI option
-`-ldflags "-X webservice/configuration.version=${VERSION}"`.
-For more information, please refer to the [Makefile](./Makefile).
+- **Cluster Umgebung**:

+  - **Google Kubernetes Engine Cluster**: `devops24-autopilot-cluster`
+  - **Netzwerk**: `devops24-network`
+  - **Subnetz**: `devops24-subnetwork`
+  - **Kubernetes Deployment**: Deployment mit drei Replikaten
+  - **LoadBalancer-Service**: Lastverteilung auf mehrere Pods oder Dienste im Cluster
+  - **Ingress-Controller (Nginx)**: Routen von eingehendem HTTP(S)-Verkehr im Cluster

-#### Run:
+- **Datenbank**: Für die Persistenz wird Redis verwendet, welches in einer separaten Compute Engine-Instanz innerhalb der GCP-Umgebung betrieben wird.

-```bash
-HOST=0.0.0.0 PORT=8080 ./artifact.bin
-```
+### CI/CD Pipeline

+Die Konfiguration für den Build des Webservices erfolgt in der GitLab CI/CD-Pipeline. Diese Pipeline unterscheidet die Stages Test, Build und Publish und wird bei Merge-Requests oder Pushes auf die Branches `main`, `dev` oder `prod` aktiviert.

-#### Interact:
+- **Test**: Durchführung von Tests mit dem Docker-Image `golang:1.21`
+- **Build**: Erstellung des Webservice-Binaries für `linux_amd64` und Speicherung der Artefakte in der Container Registry
+- **Publish**: Veröffentlichung des Docker-Images in der Container Registry mit Versionierung auf der Basis der Pipeline-ID

-##### Landing page 
+## Lifecycle

-plain text:
-```bash
-curl http://localhost:8080
-```
+![Lifecycle](Lifecycle.png)

-HTML:
-```bash
-curl --header 'Accept: text/html; charset=utf-8' http://localhost:8080
-# or just open in a browser
-```
+- **Pipeline Trigger**: Die Pipeline wird durch Merge-Requests oder Pushes auf den Branches `main`, `dev` oder `prod` getriggert. Nach einem erfolgreichen Durchlauf der Pipeline wird ein Container-Image als Artefakt erstellt.
+- **Artifakte**: Ein Commit auf den `dev`-Branch erzeugt ein mit `dev` getaggtes Artefakt, das für Tests in der Entwicklungsumgebung verwendet wird. Diese Tests erfolgen durch lokale Ausführung mittels Podman. Wenn alles in Ordnung ist, kann ein Commit auf den `prod`-Branch den Build eines Container-Images auslösen. Dieses wird anschließend von Kubernetes im Google Cloud Cluster anhand des `prod`-Tags erkannt und in Pods ausgeführt.

+## Weitere Konfigurationen

-##### Health check
-
-```bash
-curl http://localhost:8080/health
-```
-
-
-##### Server side environment variables
-
-List environment variables visible by the webservice process if environment
-is not `production`.
-
-```bash
-curl http://localhost:8080/env
-```
-
-
-##### State life cycle
-
-URL slug is used as identifier and the body is the actual *data* being stored.
-Please note, when writing (add or change) something, `Content-Type` must be set
-in the request header.
-
-Write an entry:
-```bash
-curl \
-  -X PUT \
-  --header 'Content-Type: text/plain; charset=utf-8' \
-  --data 'foo' \
-  http://localhost:8080/state/bar
-```
-
-Find out MIME type and size of an entry:
-```bash
-curl \
-  -X HEAD \
-  http://localhost:8080/state/bar
-```
-
-Obtain an entry:
-```bash
-curl \
-  -X GET \
-  http://localhost:8080/state/bar
-```
-
-Remove an entry:
-```bash
-curl \
-  -X DELETE \
-  --verbose \
-  http://localhost:8080/state/bar
-```
-
-List all existing entries (returns JSON or plain text, depending on the `Accept` header):
-```bash
-curl \
-  -X GET \
-  --header 'Accept: text/plain' \
-  http://localhost:8080/states
-```
-
-Upload an entire file:
-```bash
-curl \
-  -X PUT \
-  --header 'Content-Type: application/pdf' \
-  --upload-file ./example.pdf \
-  http://localhost:8080/state/pdf-doc
-```
-
-Download a file:
-```bash
-curl \
-  -X GET \
-  --output ./example-copy.pdf \
-  http://localhost:8080/state/pdf-doc
-```
+- **Monitoring**: Die Anwendung wird mithilfe von GCP-Monitoring überwacht.
+- **TLS**: Wird wie DNS nur in der Entwicklungsumgebung konfiguriert. Durch ein selbstsigniertes Zertifikat erfolgt die Kommunikation über Nginx, welches den HTTPS-Traffic auf Port 8443 leitet.
+- **DNS**: In der `/etc/hosts`-Datei konfiguriert; es wird überprüft, ob die Seite über einen Alias per HTTPS verfügbar ist.
--- a/gcp-cluster/app.tf
+++ b/gcp-cluster/app.tf
+data "google_client_config" "default" {}
+
+provider "kubernetes" {
+  host                   = "https://${google_container_cluster.default.endpoint}"
+  token                  = data.google_client_config.default.access_token
+  cluster_ca_certificate = base64decode(google_container_cluster.default.master_auth[0].cluster_ca_certificate)
+
+  ignore_annotations = [
+    "^autopilot\\.gke\\.io\\/.*",
+    "^cloud\\.google\\.com\\/.*"
+  ]
+}
+
+resource "kubernetes_deployment_v1" "default" {
+  metadata {
+    name = "devops24-webservice-deployment"
+  }
+
+  spec {
+    replicas = 3
+
+    selector {
+      match_labels = {
+        app = "devops24-webservice"
+      }
+    }
+
+    template {
+      metadata {
+        labels = {
+          app = "devops24-webservice"
+        }
+      }
+
+      spec {
+        container {
+          image = "registry.bht-berlin.de:443/masi9606/webservice:0.0.36"
+          name  = "devops24-webservice-container"
+
+          port {
+            container_port = 8080
+            name           = "devops24-svc"
+          }
+
+          env {
+            name  = "HOST"
+            value = "0.0.0.0"
+          }
+
+          env {
+            name  = "PORT"
+            value = "8080"
+          }
+
+          security_context {
+            allow_privilege_escalation = false
+            privileged                 = false
+            read_only_root_filesystem  = false
+
+            capabilities {
+              add  = []
+              drop = ["NET_RAW"]
+            }
+          }
+
+          liveness_probe {
+            http_get {
+              path = "/health"
+              port = "devops24-svc"
+
+              http_header {
+                name  = "X-Custom-Header"
+                value = "Awesome"
+              }
+            }
+
+            initial_delay_seconds = 3
+            period_seconds        = 3
+          }
+        }
+
+        security_context {
+          run_as_non_root = true
+
+          seccomp_profile {
+            type = "RuntimeDefault"
+          }
+        }
+
+        # Toleration is currently required to prevent perpetual diff:
+        # https://github.com/hashicorp/terraform-provider-kubernetes/pull/2380
+        toleration {
+          effect   = "NoSchedule"
+          key      = "kubernetes.io/arch"
+          operator = "Equal"
+          value    = "amd64"
+        }
+      }
+    }
+  }
+}
+
+resource "kubernetes_service_v1" "default" {
+  metadata {
+    name = "devops24-webservice-loadbalancer"
+    annotations = {
+      "cloud.google.com/l4-rbs" = "enabled"
+    }
+  }
+
+  spec {
+    selector = {
+      app = "devops24-webservice"
+    }
+
+    ip_family_policy = "RequireDualStack"
+
+    port {
+      name       = "http"
+      port       = 80
+      target_port = "devops24-svc"
+      protocol   = "TCP"
+    }
+
+    type = "LoadBalancer"
+  }
+
+  depends_on = [time_sleep.wait_service_cleanup]
+}
+
+resource "kubernetes_ingress_v1" "default" {
+  metadata {
+    name = "devops24-webservice-ingress"
+    annotations = {
+      "kubernetes.io/ingress.class" = "nginx"
+    }
+  }
+
+  spec {
+    rule {
+      http {
+        path {
+          path      = "/"
+          path_type = "Prefix"
+
+          backend {
+            service {
+              name = kubernetes_service_v1.default.metadata[0].name
+              port {
+                number = 80
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+# Provide time for Service cleanup
+resource "time_sleep" "wait_service_cleanup" {
+  depends_on = [google_container_cluster.default]
+
+  destroy_duration = "180s"
+}
--- a/gcp-cluster/cluster.tf
+++ b/gcp-cluster/cluster.tf
+resource "google_compute_network" "default" {
+  name = "devops24-network"
+
+  auto_create_subnetworks  = false
+  enable_ula_internal_ipv6 = true
+}
+
+resource "google_compute_subnetwork" "default" {
+  name = "devops24-subnetwork"
+
+  ip_cidr_range = "10.0.0.0/16"
+  region        = "europe-west3"
+
+  stack_type       = "IPV4_IPV6"
+  ipv6_access_type = "EXTERNAL"
+
+  network = google_compute_network.default.id
+  secondary_ip_range {
+    range_name    = "services-range"
+    ip_cidr_range = "192.168.0.0/24"
+  }
+
+  secondary_ip_range {
+    range_name    = "pod-ranges"
+    ip_cidr_range = "192.168.1.0/24"
+  }
+}
+
+resource "google_container_cluster" "default" {
+  name = "devops24-autopilot-cluster"
+
+  location                 = "europe-west3"
+  enable_autopilot         = true
+  enable_l4_ilb_subsetting = true
+
+  network    = google_compute_network.default.id
+  subnetwork = google_compute_subnetwork.default.id
+
+  ip_allocation_policy {
+    stack_type                    = "IPV4_IPV6"
+    services_secondary_range_name = google_compute_subnetwork.default.secondary_ip_range[0].range_name
+    cluster_secondary_range_name  = google_compute_subnetwork.default.secondary_ip_range[1].range_name
+  }
+
+  # Set `deletion_protection` to `true` will ensure that one cannot
+  # accidentally delete this instance by use of Terraform.
+  deletion_protection = false
+}
--- a/gcp-cluster/deploy.sh
+++ b/gcp-cluster/deploy.sh
+#kubectl config current-context
+#kubectl api-resources
+#kubectl get pods
+
+#init
+kubectl config set-context --current --namespace=default
+
+#tofu
+tofu init
+tofu plan
+tofu apply
+
+#connect
+gcloud container clusters get-credentials devops24-autopilot-cluster --region europe-west3 --project devops24-sose
+
+#debug
+kubectl get pods --output wide
+kubectl get services --output wide
+kubectl get ingress --output wide
+
+kubectl describe pod devops24-webservice-deployment-c4d7b6876-rdw6m
+
+#hop in
+kubectl exec --stdin --tty devops24-webservice-deployment-c4d7b6876-rdw6m -- bash
+
+#replace
+kubectl delete pod devops24-webservice-deployment-c4d7b6876-rdw6m
+kubectl get pods --watch
+
+# get hello world
+curl http://104.199.1.99
+curl http://104.199.1.99/health
+
+#destroy
+tofu destroy
\ No newline at end of file
--- a/gcp-cluster/provider.tf
+++ b/gcp-cluster/provider.tf
+provider "google" {
+  project     = "devops24-sose"
+  region      = "europe-west3"
+  credentials = file("/home/schnarkus/.gcp/keyfile.json")
+}
--- a/local-environment/default.conf
+++ b/local-environment/default.conf
+server {
+    listen 8443 ssl;
+    server_name webservice.local;
+
+    ssl_certificate /etc/nginx/certs/nginx.crt;
+    ssl_certificate_key /etc/nginx/certs/nginx.key;
+
+    location / {
+        proxy_pass http://localhost:8080;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+    }
+}
--- a/local-environment/prometheus.yml
+++ b/local-environment/prometheus.yml
+global:
+  scrape_interval: 15s
+
+scrape_configs:
+  - job_name: "prometheus"
+    scrape_interval: 5s
+    static_configs:
+      - targets: ["localhost:9090"]
+
+  - job_name: "webservice"
+    scrape_interval: 15s
+    static_configs:
+      - targets: ["10.89.0.2:8080"]
--- a/local-environment/setup-local.sh
+++ b/local-environment/setup-local.sh
+#!/bin/bash
+
+# webservice container
+IMAGE_NAME="registry.bht-berlin.de:443/masi9606/webservice:dev"
+CONTAINER_NAME="webservice_container"
+CONTAINER_HTTP_PORT="8080"
+CONTAINER_HTTPS_PORT="8443"
+NGINX_CONF="/etc/nginx/sites-available/default"
+CERT_DIR="/etc/nginx/certs"
+WEB_SERVICE_DOMAIN="webservice.local" 
+
+# prometheus container
+PROMETHEUS_IMAGE="prom/prometheus:latest"
+PROMETHEUS_CONTAINER_NAME="prometheus_monitor"
+PROMETHEUS_PORT="9090"
+PROMETHEUS_DOMAIN="monitoring.local" 
+PROMETHEUS_CONFIG_FILE="prometheus.yml"
+
+# local network
+NETWORK_NAME="dev_network"
+
+# Stop old containers
+podman stop $CONTAINER_NAME
+podman rm -f $CONTAINER_NAME
+podman stop $PROMETHEUS_CONTAINER_NAME
+podman rm -f $PROMETHEUS_CONTAINER_NAME
+
+# Pull images
+podman pull $IMAGE_NAME
+podman pull $PROMETHEUS_IMAGE
+
+# Run webservice container
+podman run -d \
+  --name "$CONTAINER_NAME" \
+  --network "$NETWORK_NAME" \
+  --ip "10.89.0.2" \
+  -p "$CONTAINER_HTTP_PORT:$CONTAINER_HTTP_PORT" \
+  -p "$CONTAINER_HTTPS_PORT:$CONTAINER_HTTPS_PORT" \
+  "$IMAGE_NAME"
+
+# Install packages
+podman exec -it $CONTAINER_NAME apt-get update
+podman exec -it $CONTAINER_NAME apt-get install -y openssl nginx systemctl curl
+
+# Generate SSL certificate and key
+podman exec -it $CONTAINER_NAME bash -c "mkdir -p $CERT_DIR && \
+    openssl req -x509 -nodes -days 365 -newkey rsa:4096 \
+    -keyout $CERT_DIR/nginx.key -out $CERT_DIR/nginx.crt \
+    -subj '/CN=$WEB_SERVICE_DOMAIN'"
+
+podman cp default.conf $CONTAINER_NAME:$NGINX_CONF
+
+# Restart Nginx to apply the changes
+podman exec -it $CONTAINER_NAME systemctl restart nginx
+
+# Wait
+podman exec -it $CONTAINER_NAME sleep 5
+
+# Add entry to /etc/hosts
+podman exec -it $CONTAINER_NAME bash -c "echo '127.0.0.1 $WEB_SERVICE_DOMAIN' >> /etc/hosts"
+
+echo "Testing from inside..."
+podman exec -it $CONTAINER_NAME curl -k https://$WEB_SERVICE_DOMAIN:$CONTAINER_HTTPS_PORT/
+
+echo "Testing from outside..."
+sudo bash -c "echo '127.0.0.1 $WEB_SERVICE_DOMAIN' >> /etc/hosts"
+curl -k https://$WEB_SERVICE_DOMAIN:$CONTAINER_HTTPS_PORT/
+
+echo "Testing Redis..."
+curl -k -X PUT --header 'Content-Type: text/plain; charset=utf-8' --data 'foo' https://$WEB_SERVICE_DOMAIN:$CONTAINER_HTTPS_PORT/state/bar
+
+# Check the value in the database
+podman exec -it $CONTAINER_NAME redis-cli KEYS '*'
+
+# Run Prometheus container
+podman run -d \
+  --name $PROMETHEUS_CONTAINER_NAME \
+  --network $NETWORK_NAME --ip "10.89.0.3" \
+  -p $PROMETHEUS_PORT:$PROMETHEUS_PORT \
+  -v $PWD/$PROMETHEUS_CONFIG_FILE:/etc/prometheus/prometheus.yml:Z \
+  $PROMETHEUS_IMAGE --config.file=/etc/prometheus/prometheus.yml
+
+# Test Prometheus
+sudo bash -c "echo '127.0.0.1 $PROMETHEUS_DOMAIN' >> /etc/hosts"
+curl http://$PROMETHEUS_DOMAIN:$PROMETHEUS_PORT/
\ No newline at end of file
No results found