From df8eea778ce178f424e16e37f3940d643fee45e0 Mon Sep 17 00:00:00 2001 From: savagebidoof Date: Sat, 14 Oct 2023 12:11:34 +0200 Subject: [PATCH] Documented examples from `11-Fault_Injection`: - 01-FaultInjection-delay - 02-FaultInjection-abort Added a section README.md to the section: - 11-Fault_Injection --- .../Deployment.yaml} | 15 -- .../01-FaultInjection-delay/Gateway.yaml | 14 + .../01-FaultInjection-delay/README.md | 251 ++++++++++++++++++ .../01-FaultInjection-delay/Service.yaml | 13 + .../VirtualService.yaml} | 20 +- .../Deployment.yaml} | 15 -- .../02-FaultInjection-abort/Gateway.yaml | 14 + .../02-FaultInjection-abort/README.md | 240 +++++++++++++++++ .../02-FaultInjection-abort/Service.yaml | 13 + .../VirtualService.yaml} | 20 +- .../05a-FaultInjection-delay/README.md | 7 - .../05b-FaultInjection-abort/README.md | 11 - 11-Fault_Injection/README.md | 9 + 13 files changed, 558 insertions(+), 84 deletions(-) rename 11-Fault_Injection/{05b-FaultInjection-abort/deployment.yaml => 01-FaultInjection-delay/Deployment.yaml} (63%) create mode 100755 11-Fault_Injection/01-FaultInjection-delay/Gateway.yaml create mode 100755 11-Fault_Injection/01-FaultInjection-delay/README.md create mode 100644 11-Fault_Injection/01-FaultInjection-delay/Service.yaml rename 11-Fault_Injection/{05a-FaultInjection-delay/gateway.yaml => 01-FaultInjection-delay/VirtualService.yaml} (50%) mode change 100755 => 100644 rename 11-Fault_Injection/{05a-FaultInjection-delay/deployment.yaml => 02-FaultInjection-abort/Deployment.yaml} (63%) create mode 100755 11-Fault_Injection/02-FaultInjection-abort/Gateway.yaml create mode 100755 11-Fault_Injection/02-FaultInjection-abort/README.md create mode 100644 11-Fault_Injection/02-FaultInjection-abort/Service.yaml rename 11-Fault_Injection/{05b-FaultInjection-abort/gateway.yaml => 02-FaultInjection-abort/VirtualService.yaml} (50%) mode change 100755 => 100644 delete mode 100755 11-Fault_Injection/05a-FaultInjection-delay/README.md delete mode 100755 11-Fault_Injection/05b-FaultInjection-abort/README.md create mode 100644 11-Fault_Injection/README.md diff --git a/11-Fault_Injection/05b-FaultInjection-abort/deployment.yaml b/11-Fault_Injection/01-FaultInjection-delay/Deployment.yaml similarity index 63% rename from 11-Fault_Injection/05b-FaultInjection-abort/deployment.yaml rename to 11-Fault_Injection/01-FaultInjection-delay/Deployment.yaml index 01dd2b0..233fd2c 100755 --- a/11-Fault_Injection/05b-FaultInjection-abort/deployment.yaml +++ b/11-Fault_Injection/01-FaultInjection-delay/Deployment.yaml @@ -1,18 +1,3 @@ -# https://github.com/istio/istio/blob/master/samples/helloworld/helloworld.yaml -apiVersion: v1 -kind: Service -metadata: - name: helloworld - labels: - app: helloworld - service: helloworld -spec: - ports: - - port: 80 - name: http - selector: - app: helloworld ---- apiVersion: apps/v1 kind: Deployment metadata: diff --git a/11-Fault_Injection/01-FaultInjection-delay/Gateway.yaml b/11-Fault_Injection/01-FaultInjection-delay/Gateway.yaml new file mode 100755 index 0000000..bbd9d67 --- /dev/null +++ b/11-Fault_Injection/01-FaultInjection-delay/Gateway.yaml @@ -0,0 +1,14 @@ +apiVersion: networking.istio.io/v1alpha3 +kind: Gateway +metadata: + name: helloworld-gateway +spec: + selector: + istio: ingressgateway # use istio default controller + servers: + - port: + number: 80 + name: http + protocol: HTTP + hosts: + - "*" \ No newline at end of file diff --git a/11-Fault_Injection/01-FaultInjection-delay/README.md b/11-Fault_Injection/01-FaultInjection-delay/README.md new file mode 100755 index 0000000..d319472 --- /dev/null +++ b/11-Fault_Injection/01-FaultInjection-delay/README.md @@ -0,0 +1,251 @@ +--- +gitea: none +include_toc: true +--- + +# Description + +Based on the [previous example](../../01-Getting_Started/01-hello_world_1_service_1_deployment), we configure a "fault" that will make the backend take 10 more seconds before receiving the request. + +This will be applied to a 90% of the incoming traffic that matches the rule and will allow to confirm in a secure environment how the application would behave in such difficult situations, and apply the modifications required to avoid issue in case there would be a network issue. + + + +This example configures: + + Generic Kubernetes resources: + - 1 Service + - 1 Deployments + + Istio resources: + - 1 Gateway + - 1 Virtual Service + + +# Based on + +- [01-hello_world_1_service_1_deployment](../../01-Getting_Started/01-hello_world_1_service_1_deployment) +- https://istio.io/latest/docs/reference/config/networking/virtual-service/#HTTPFaultInjection-Delay + +# Configuration + +## Service + +Creates a service named `helloworld`. + +This service listens for the port `80` expecting `HTTP` traffic and will forward the incoming traffic towards the port `80` from the destination pod. + +```yaml +apiVersion: v1 +kind: Service +metadata: + name: helloworld + labels: + app: helloworld + service: helloworld +spec: + ports: + - port: 80 + name: http + selector: + app: helloworld +``` + +## Deployment + +Deploys a Nginx server that listens for the port `80`. + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: helloworld-nginx + labels: + app: helloworld +spec: + replicas: 1 + selector: + matchLabels: + app: helloworld + template: + metadata: + labels: + app: helloworld + spec: + containers: + - name: helloworld + image: nginx + resources: + requests: + cpu: "100m" + imagePullPolicy: IfNotPresent #Always + ports: + - containerPort: 80 +``` + +## Gateway + +Deploys an Istio gateway that's listening to the port `80` for `HTTP` traffic. + +It doesn't filter for any specific host. + +The `selector` field is used to "choose" which Istio Load Balancers will have this gateway assigned to. + +The Istio `default` profile creates a Load Balancer in the namespace `istio-system` that has the label `istio: ingressgateway` set, allowing us to target that specific Load Balancer and assign this gateway resource to it. + +```yaml +apiVersion: networking.istio.io/v1alpha3 +kind: Gateway +metadata: + name: helloworld-gateway +spec: + selector: + istio: ingressgateway # use istio default controller + servers: + - port: + number: 80 + name: http + protocol: HTTP + hosts: + - "*" +``` + +## VirtualService + +The Virtual Service resources are used to route and filter the received traffic from the gateway resources, and route it towards the desired destination. + +On this example we select the gateway `helloworld-gateway`, which is the [gateway that 's described in the `Gateway` section](#gateway). + +On this resource, we are also not limiting the incoming traffic to any specific host, allowing for all the incoming traffic to go through the rules set. + +Here we created a rule that will be applied on `HTTP` related traffic (including `HTTPS` and `HTTP2`) when the destination path is exactly `/helloworld`. + +This traffic will be forwarded to the port `80` of the destination service `helloworld` (the full path URL equivalent would be `helloworld.$NAMESPACE.svc.cluster.local`). + +There will be an internal URL rewrite set, as if the URL is not modified, it would attempt to reach to the `/helloworld` path from the Nginx deployment, which currently has no content and would result in an error code `404` (Not found). + +Additionally, we apply a "fault", where a 90% of the traffic will have 10 seconds extra of delay. + +```yaml +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: helloworld-vs +spec: + hosts: + - "*" + gateways: + - helloworld-gateway + http: + - match: + - uri: + exact: /helloworld + route: + - destination: + host: helloworld + port: + number: 80 + rewrite: + uri: "/" + fault: + delay: + percentage: + value: 90 + fixedDelay: 10s +``` + +# Walkthrough + +## Deploy resources + +Deploy the resources. + +```shell +kubectl apply -f ./ +``` +```text +deployment.apps/helloworld-nginx created +gateway.networking.istio.io/helloworld-gateway created +service/helloworld created +virtualservice.networking.istio.io/helloworld-vs created +``` + +## Wait for the pods to be ready + +Wait for the Nginx deployments to be up and ready. + +```shell +kubectl get deployment helloworld-nginx -w +``` +```text +NAME READY UP-TO-DATE AVAILABLE AGE +helloworld-nginx 1/1 1 1 12s +``` + +## Test the service + +### Get LB IP + +To perform the desired tests, we will need to obtain the IP Istio Load Balancer that we selected in the [Gateway section](#gateway). + +On my environment, the IP is the `192.168.1.50`. + +```shell +kubectl get svc -l istio=ingressgateway -A +``` +```text +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +istio-ingressgateway LoadBalancer 10.97.47.216 192.168.1.50 15021:31316/TCP,80:32012/TCP,443:32486/TCP 39h +``` + +### helloworld + +We will use the `curl` command and feed it a template to provide us with some timings and as well of confirming the status code from the request. + +Since the fault that we set had a 90% chance of triggering, if you are "unlucky", and get instantly the response from the backend, you might need to run the command multiple times in order to get the fault triggered. + +```shell +curl -w @- -o /dev/null -s 192.168.1.50/helloworld <<'EOF' + http_code: %{http_code}\n + time_namelookup: %{time_namelookup}\n + time_connect: %{time_connect}\n + time_appconnect: %{time_appconnect}\n + time_pretransfer: %{time_pretransfer}\n + time_redirect: %{time_redirect}\n + time_starttransfer: %{time_starttransfer}\n + ----------\n + time_total: %{time_total}\n +EOF +``` + +```text + http_code: 200 + time_namelookup: 0.000010 + time_connect: 0.000671 + time_appconnect: 0.000000 + time_pretransfer: 0.000689 + time_redirect: 0.000000 + time_starttransfer: 10.008781 + ---------- + time_total: 10.008817 +``` + +From the command output, we can observe that the request took more than 10 seconds to be replied, and as well the status code was successful, meaning that the application was able to handle the request. + +## Cleanup + +Finally, a cleanup from the resources deployed. + +```shell +kubectl delete -f ./ +``` +```text +deployment.apps "helloworld-nginx" deleted +gateway.networking.istio.io "helloworld-gateway" deleted +service "helloworld" deleted +virtualservice.networking.istio.io "helloworld-vs" deleted +``` + +# Links of interest + +- https://istio.io/latest/docs/reference/config/networking/virtual-service/#HTTPFaultInjection-Delay \ No newline at end of file diff --git a/11-Fault_Injection/01-FaultInjection-delay/Service.yaml b/11-Fault_Injection/01-FaultInjection-delay/Service.yaml new file mode 100644 index 0000000..271ce25 --- /dev/null +++ b/11-Fault_Injection/01-FaultInjection-delay/Service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: helloworld + labels: + app: helloworld + service: helloworld +spec: + ports: + - port: 80 + name: http + selector: + app: helloworld \ No newline at end of file diff --git a/11-Fault_Injection/05a-FaultInjection-delay/gateway.yaml b/11-Fault_Injection/01-FaultInjection-delay/VirtualService.yaml old mode 100755 new mode 100644 similarity index 50% rename from 11-Fault_Injection/05a-FaultInjection-delay/gateway.yaml rename to 11-Fault_Injection/01-FaultInjection-delay/VirtualService.yaml index fce0b98..b06e3c9 --- a/11-Fault_Injection/05a-FaultInjection-delay/gateway.yaml +++ b/11-Fault_Injection/01-FaultInjection-delay/VirtualService.yaml @@ -1,19 +1,3 @@ -# https://github.com/istio/istio/blob/master/samples/helloworld/helloworld-gateway.yaml -apiVersion: networking.istio.io/v1alpha3 -kind: Gateway -metadata: - name: helloworld-gateway -spec: - selector: - istio: ingressgateway # use istio default controller - servers: - - port: - number: 80 - name: http - protocol: HTTP - hosts: - - "*" ---- apiVersion: networking.istio.io/v1alpha3 kind: VirtualService metadata: @@ -35,7 +19,7 @@ spec: rewrite: uri: "/" fault: - delay: + delay: percentage: - value: 90 + value: 90 fixedDelay: 10s \ No newline at end of file diff --git a/11-Fault_Injection/05a-FaultInjection-delay/deployment.yaml b/11-Fault_Injection/02-FaultInjection-abort/Deployment.yaml similarity index 63% rename from 11-Fault_Injection/05a-FaultInjection-delay/deployment.yaml rename to 11-Fault_Injection/02-FaultInjection-abort/Deployment.yaml index 01dd2b0..233fd2c 100755 --- a/11-Fault_Injection/05a-FaultInjection-delay/deployment.yaml +++ b/11-Fault_Injection/02-FaultInjection-abort/Deployment.yaml @@ -1,18 +1,3 @@ -# https://github.com/istio/istio/blob/master/samples/helloworld/helloworld.yaml -apiVersion: v1 -kind: Service -metadata: - name: helloworld - labels: - app: helloworld - service: helloworld -spec: - ports: - - port: 80 - name: http - selector: - app: helloworld ---- apiVersion: apps/v1 kind: Deployment metadata: diff --git a/11-Fault_Injection/02-FaultInjection-abort/Gateway.yaml b/11-Fault_Injection/02-FaultInjection-abort/Gateway.yaml new file mode 100755 index 0000000..bbd9d67 --- /dev/null +++ b/11-Fault_Injection/02-FaultInjection-abort/Gateway.yaml @@ -0,0 +1,14 @@ +apiVersion: networking.istio.io/v1alpha3 +kind: Gateway +metadata: + name: helloworld-gateway +spec: + selector: + istio: ingressgateway # use istio default controller + servers: + - port: + number: 80 + name: http + protocol: HTTP + hosts: + - "*" \ No newline at end of file diff --git a/11-Fault_Injection/02-FaultInjection-abort/README.md b/11-Fault_Injection/02-FaultInjection-abort/README.md new file mode 100755 index 0000000..5d52178 --- /dev/null +++ b/11-Fault_Injection/02-FaultInjection-abort/README.md @@ -0,0 +1,240 @@ +--- +gitea: none +include_toc: true +--- + +# Description + +Based on the [previous example](../02-FaultInjection-abort), where we configure a "fault" that will make the backend take 10 more seconds before receiving the request, this time will make the request abort, resulting in a failed request (503 status code). + +This will be applied to a 90% of the incoming traffic that matches the rule and will allow to confirm in a secure environment how the application would behave in such difficult situations, and apply the modifications required to avoid issue in case there would be a network issue. + + +This example configures: + + Generic Kubernetes resources: + - 1 Service + - 1 Deployments + + Istio resources: + - 1 Gateway + - 1 Virtual Service + + +# Based on + +- [02-FaultInjection-abort](../02-FaultInjection-abort) +- https://istio.io/latest/docs/reference/config/networking/virtual-service/#HTTPFaultInjection-Abort + +# Configuration + +## Service + +Creates a service named `helloworld`. + +This service listens for the port `80` expecting `HTTP` traffic and will forward the incoming traffic towards the port `80` from the destination pod. + +```yaml +apiVersion: v1 +kind: Service +metadata: + name: helloworld + labels: + app: helloworld + service: helloworld +spec: + ports: + - port: 80 + name: http + selector: + app: helloworld +``` + +## Deployment + +Deploys a Nginx server that listens for the port `80`. + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: helloworld-nginx + labels: + app: helloworld +spec: + replicas: 1 + selector: + matchLabels: + app: helloworld + template: + metadata: + labels: + app: helloworld + spec: + containers: + - name: helloworld + image: nginx + resources: + requests: + cpu: "100m" + imagePullPolicy: IfNotPresent #Always + ports: + - containerPort: 80 +``` + +## Gateway + +Deploys an Istio gateway that's listening to the port `80` for `HTTP` traffic. + +It doesn't filter for any specific host. + +The `selector` field is used to "choose" which Istio Load Balancers will have this gateway assigned to. + +The Istio `default` profile creates a Load Balancer in the namespace `istio-system` that has the label `istio: ingressgateway` set, allowing us to target that specific Load Balancer and assign this gateway resource to it. + +```yaml +apiVersion: networking.istio.io/v1alpha3 +kind: Gateway +metadata: + name: helloworld-gateway +spec: + selector: + istio: ingressgateway # use istio default controller + servers: + - port: + number: 80 + name: http + protocol: HTTP + hosts: + - "*" +``` + +## VirtualService + +The Virtual Service resources are used to route and filter the received traffic from the gateway resources, and route it towards the desired destination. + +On this example we select the gateway `helloworld-gateway`, which is the [gateway that 's described in the `Gateway` section](#gateway). + +On this resource, we are also not limiting the incoming traffic to any specific host, allowing for all the incoming traffic to go through the rules set. + +Here we created a rule that will be applied on `HTTP` related traffic (including `HTTPS` and `HTTP2`) when the destination path is exactly `/helloworld`. + +This traffic will be forwarded to the port `80` of the destination service `helloworld` (the full path URL equivalent would be `helloworld.$NAMESPACE.svc.cluster.local`). + +There will be an internal URL rewrite set, as if the URL is not modified, it would attempt to reach to the `/helloworld` path from the Nginx deployment, which currently has no content and would result in an error code `404` (Not found). + +Additionally, we apply a "fault", where a 90% of the traffic will be aborted and receive a 503 status code. + +```yaml +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: helloworld-vs +spec: + hosts: + - "*" + gateways: + - helloworld-gateway + http: + - match: + - uri: + exact: /helloworld + route: + - destination: + host: helloworld + port: + number: 80 + rewrite: + uri: "/" + fault: + abort: + percentage: + value: 90 + httpStatus: 503 +``` + +# Walkthrough + +## Deploy resources + +Deploy the resources. + +```shell +kubectl apply -f ./ +``` +```text +deployment.apps/helloworld-nginx created +gateway.networking.istio.io/helloworld-gateway created +service/helloworld created +virtualservice.networking.istio.io/helloworld-vs created +``` + +## Wait for the pods to be ready + +Wait for the Nginx deployments to be up and ready. + +```shell +kubectl get deployment helloworld-nginx -w +``` +```text +NAME READY UP-TO-DATE AVAILABLE AGE +helloworld-nginx 1/1 1 1 12s +``` + +## Test the service + +### Get LB IP + +To perform the desired tests, we will need to obtain the IP Istio Load Balancer that we selected in the [Gateway section](#gateway). + +On my environment, the IP is the `192.168.1.50`. + +```shell +kubectl get svc -l istio=ingressgateway -A +``` +```text +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +istio-ingressgateway LoadBalancer 10.97.47.216 192.168.1.50 15021:31316/TCP,80:32012/TCP,443:32486/TCP 39h +``` + +### helloworld + +We will use the `curl` command and feed it a template to provide us with the status code from the request. + +Since the fault that we set had a 90% chance of triggering, if you are "unlucky", and get instantly the response from the backend, you might need to run the command multiple times in order to get the fault triggered. + +```shell +curl -w @- -o /dev/null -s 192.168.1.21/helloworld <<'EOF' + http_code: %{http_code}\n + ----------\n + time_total: %{time_total}\n +EOF +``` + +```text + http_code: 503 + ---------- + time_total: 0.037870 +``` + +From the command output, we can observe that the request resulted in a 503 status code. + +We could continue sending curls until we receive a successful `200` status code. + +## Cleanup + +Finally, a cleanup from the resources deployed. + +```shell +kubectl delete -f ./ +``` +```text +deployment.apps "helloworld-nginx" deleted +gateway.networking.istio.io "helloworld-gateway" deleted +service "helloworld" deleted +virtualservice.networking.istio.io "helloworld-vs" deleted +``` + +# Links of interest + +- https://istio.io/latest/docs/reference/config/networking/virtual-service/#HTTPFaultInjection-Abort \ No newline at end of file diff --git a/11-Fault_Injection/02-FaultInjection-abort/Service.yaml b/11-Fault_Injection/02-FaultInjection-abort/Service.yaml new file mode 100644 index 0000000..271ce25 --- /dev/null +++ b/11-Fault_Injection/02-FaultInjection-abort/Service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: helloworld + labels: + app: helloworld + service: helloworld +spec: + ports: + - port: 80 + name: http + selector: + app: helloworld \ No newline at end of file diff --git a/11-Fault_Injection/05b-FaultInjection-abort/gateway.yaml b/11-Fault_Injection/02-FaultInjection-abort/VirtualService.yaml old mode 100755 new mode 100644 similarity index 50% rename from 11-Fault_Injection/05b-FaultInjection-abort/gateway.yaml rename to 11-Fault_Injection/02-FaultInjection-abort/VirtualService.yaml index d6865b4..d0852a1 --- a/11-Fault_Injection/05b-FaultInjection-abort/gateway.yaml +++ b/11-Fault_Injection/02-FaultInjection-abort/VirtualService.yaml @@ -1,19 +1,3 @@ -# https://github.com/istio/istio/blob/master/samples/helloworld/helloworld-gateway.yaml -apiVersion: networking.istio.io/v1alpha3 -kind: Gateway -metadata: - name: helloworld-gateway -spec: - selector: - istio: ingressgateway # use istio default controller - servers: - - port: - number: 80 - name: http - protocol: HTTP - hosts: - - "*" ---- apiVersion: networking.istio.io/v1alpha3 kind: VirtualService metadata: @@ -35,7 +19,7 @@ spec: rewrite: uri: "/" fault: - abort: + abort: percentage: - value: 90 + value: 90 httpStatus: 503 \ No newline at end of file diff --git a/11-Fault_Injection/05a-FaultInjection-delay/README.md b/11-Fault_Injection/05a-FaultInjection-delay/README.md deleted file mode 100755 index 081d8fc..0000000 --- a/11-Fault_Injection/05a-FaultInjection-delay/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Continues from - -- 01-hello_world_1_service_1_deployment - - -https://istio.io/latest/docs/reference/config/networking/virtual-service/#HTTPFaultInjection-Delay - diff --git a/11-Fault_Injection/05b-FaultInjection-abort/README.md b/11-Fault_Injection/05b-FaultInjection-abort/README.md deleted file mode 100755 index b69a0af..0000000 --- a/11-Fault_Injection/05b-FaultInjection-abort/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# Continues from - -- 05a-FaultInjection-delay - - -https://istio.io/latest/docs/reference/config/networking/virtual-service/#HTTPFaultInjection-Abort - - - -curl 192.168.1.50/helloworld -I - diff --git a/11-Fault_Injection/README.md b/11-Fault_Injection/README.md new file mode 100644 index 0000000..9965ee0 --- /dev/null +++ b/11-Fault_Injection/README.md @@ -0,0 +1,9 @@ + +## Description + +This section focuses on applying configurations to test the resiliency from the deployed applications. + +## Examples + +- 01-FaultInjection-delay +- 02-FaultInjection-abort