Red Hat build of MicroShift

$ sudo cp /etc/microshift/ovn.yaml.default /etc/microshift/ovn.yaml

$ cat /etc/microshift/ovn.yaml

mtu: 1400

mtu: 1300

$ export KUBECONFIG=$PWD/kubeconfig

$ pod=$(oc get pods -n openshift-ovn-kubernetes | awk -F " " '/ovnkube-master/{print $1}')

$ oc -n openshift-ovn-kubernetes delete pod $pod

$ oc get pods -n openshift-ovn-kubernetes

dns:
 hosts:
   status: <Enabled|Disabled>
   file: <filepath>

$ sudo systemctl restart microshift

apiServer:
# ...
network:
  clusterNetwork:
  - fd01::/48
  serviceNetwork:
  - fd02::/112
node:
  nodeIP: 2600:1f14:1c48:ee00:2d76:3190:5bc2:5aef
# ...

$ sudo systemctl start microshift

$ oc get node -o jsonpath='{.items[].spec.podCIDRs[]}'

fd01::/48

$ oc get pod -A -o wide

NAMESPACE                  NAME                                      READY   STATUS    RESTARTS   AGE   IP                      NODE           NOMINATED NODE   READINESS GATES
kube-system                csi-snapshot-controller-bb7cb654b-rqrt6   1/1     Running   0          65s   fd01:0:0:1::5           microshift-9   <none>           <none>
openshift-dns              dns-default-cjn66                         2/2     Running   0          62s   fd01:0:0:1::9           microshift-9   <none>           <none>
openshift-dns              node-resolver-ppnjb                       1/1     Running   0          63s   2001:db9:ca7:ff::1db8   microshift-9   <none>           <none>
openshift-ingress          router-default-6d97d7b8b6-wdtmg           1/1     Running   0          61s   fd01:0:0:1::8           microshift-9   <none>           <none>
openshift-ovn-kubernetes   ovnkube-master-gfvp5                      4/4     Running   0          63s   2001:db9:ca7:ff::1db8   microshift-9   <none>           <none>
openshift-ovn-kubernetes   ovnkube-node-bnpjh                        1/1     Running   0          63s   2001:db9:ca7:ff::1db8   microshift-9   <none>           <none>
openshift-service-ca       service-ca-5d7bd9db6-j25bd                1/1     Running   0          60s   fd01:0:0:1::4           microshift-9   <none>           <none>
openshift-storage          lvms-operator-656cd9b59b-bwr47            1/1     Running   0          63s   fd01:0:0:1::7           microshift-9   <none>           <none>
openshift-storage          vg-manager-f7dmk                          1/1     Running   0          27s   fd01:0:0:1::a           microshift-9   <none>           <none>

$ oc get svc -A

NAMESPACE           NAME                            TYPE           CLUSTER-IP   EXTERNAL-IP                                             PORT(S)                      AGE
default             kubernetes                      ClusterIP      fd02::1      <none>                                                  443/TCP                      3m42s
openshift-dns       dns-default                     ClusterIP      fd02::a      <none>                                                  53/UDP,53/TCP,9154/TCP       2m58s
openshift-ingress   router-default                  LoadBalancer   fd02::f2e6   2001:db9:ca7:ff::1db8,fd01:0:0:1::2,fd02::1:0,fd69::2   80:31133/TCP,443:31996/TCP   2m58s
openshift-ingress   router-internal-default         ClusterIP      fd02::c55e   <none>                                                  80/TCP,443/TCP,1936/TCP      2m58s
openshift-storage   lvms-operator-metrics-service   ClusterIP      fd02::7afb   <none>                                                  443/TCP                      2m58s
openshift-storage   lvms-webhook-service            ClusterIP      fd02::d8dd   <none>                                                  443/TCP                      2m58s
openshift-storage   vg-manager-metrics-service      ClusterIP      fd02::fc1    <none>                                                  443/TCP                      2m58s

apiServer:
# ...
apiServer:
  subjectAltNames:
  - 192.168.113.117
  - 2001:db9:ca7:ff::1db8
network:
  clusterNetwork:
  - 10.42.0.0/16
  - fd01::/48
  serviceNetwork:
  - 10.43.0.0/16
  - fd02::/112
node:
  nodeIP: 192.168.113.117
  nodeIPv6: 2001:db9:ca7:ff::1db8
# ...

$ sudo systemctl start microshift

$ sudo systemctl restart microshift

$ oc get pod -A -o wide

NAMESPACE                  NAME                                      READY   STATUS    RESTARTS        AGE     IP                NODE           NOMINATED NODE   READINESS GATES
kube-system                csi-snapshot-controller-bb7cb654b-7s5ql   1/1     Running   0               46m     10.42.0.6         microshift-9   <none>           <none>
openshift-dns              dns-default-zxkqn                         2/2     Running   0               46m     10.42.0.5         microshift-9   <none>           <none>
openshift-dns              node-resolver-r2h5z                       1/1     Running   0               46m     192.168.113.117   microshift-9   <none>           <none>
openshift-ingress          router-default-5b75594b4-228z7            1/1     Running   0               2m5s    10.42.0.3         microshift-9   <none>           <none>
openshift-ovn-kubernetes   ovnkube-master-bltk7                      4/4     Running   2 (2m32s ago)   2m36s   192.168.113.117   microshift-9   <none>           <none>
openshift-ovn-kubernetes   ovnkube-node-9ghgs                        1/1     Running   2 (2m32s ago)   46m     192.168.113.117   microshift-9   <none>           <none>
openshift-service-ca       service-ca-5d7bd9db6-qgwgw                1/1     Running   0               46m     10.42.0.7         microshift-9   <none>           <none>
openshift-storage          lvms-operator-656cd9b59b-8rpf4            1/1     Running   0               46m     10.42.0.8         microshift-9   <none>           <none>
openshift-storage          vg-manager-wqmh4                          1/1     Running   2 (2m39s ago)   46m     10.42.0.10        microshift-9   <none>           <none>

$ oc get pod -n openshift-ovn-kubernetes ovnkube-master-bltk7 -o jsonpath='{.status.podIPs}'

[{"ip":"192.168.113.117"},{"ip":"2001:db9:ca7:ff::1db8"}]

$ oc get pod -n openshift-ingress router-default-5b75594b4-228z7 -o jsonpath='{.status.podIPs}'

[{"ip":"10.42.0.3"},{"ip":"fd01:0:0:1::3"}]

kind: Service
apiVersion: v1
metadata:
  name: microshift-new-service
  labels: app: microshift-application
spec:
  type: NodePort
  ipFamilyPolicy: PreferDualStack
# ...

[Service]
Environment="http_proxy=http://$PROXY_USER:$PROXY_PASSWORD@$PROXY_SERVER:$PROXY_PORT/"

apiServer:
  featureGates:
    featureSet: "CustomNoUpgrade"
    customNoUpgrade:
      enabled:
      - "CPUManagerPolicyBetaOptions"
kubelet:
  reservedSystemCPUs: "0"
  cpuManagerPolicy: static
  cpuManagerPolicyOptions:
    prefer-align-cpus-by-uncorecache: "true"

$ sudo systemctl restart microshift

# Isolate cores 2-7 for running application workloads
isolated_cores=2-7

# Size of the hugepages
hugepages_size=2M

# Number of hugepages
hugepages=0

# Additional kernel arguments
additional_args=

# CPU set to be offlined
offline_cpu_set=

$ sudo tuned-adm profile microshift-baseline

profile: microshift-baseline
reboot_after_apply: True

$ sudo systemctl enable microshift-tuned.service

$ sudo systemctl start microshift-tuned.service

$ sudo subscription-manager repos --enable rhel-9-for-x86_64-rt-rpms

$ sudo dnf install -y kernel-rt

$ RTVER=$(rpm -q --queryformat '%{version}-%{release}.%{arch}' kernel-rt | sort | tail -1)

$ sudo grubby --set-default="/boot/vmlinuz-${RTVER}+rt"

$ sudo cp /etc/microshift/ovn.yaml.default /etc/microshift/ovn.yaml

$ cat /etc/microshift/ovn.yaml

mtu: 1400

mtu: 1300

apiVersion: sriovnetwork.openshift.io/v1
kind: SriovNetworkNodePolicy
metadata:
  name: policyoneflag
  namespace: sriov-network-operator
spec:
  resourceName: policyoneflag
  nodeSelector:
    node.kubernetes.io/instance-type: rhde
  priority: 10
  numVfs: 5
  nicSelector:
    pfNames: ["ens5"]
  deviceType: "netdevice"
  isRdma: false

$ oc create -f policyoneflag-sriov-node-network.yaml

$ oc get sriovnetworknodestates -n sriov-network-operator <node_name> -o jsonpath='{.status.syncStatus}'

apiVersion: sriovnetwork.openshift.io/v1
kind: SriovNetwork
metadata:
  name: onevalidflag
  namespace: sriov-network-operator
spec:
  resourceName: policyoneflag
  networkNamespace: sysctl-tuning-test
  ipam: '{ "type": "static" }'
  capabilities: '{ "mac": true, "ips": true }'
  metaPlugins : |
    {
      "type": "tuning",
      "capabilities":{
        "mac":true
      },
      "sysctl":{
         "net.ipv4.conf.IFNAME.accept_redirects": "1"
      }
    }

$ oc create -f sriov-network-interface-sysctl.yaml

$ oc get network-attachment-definitions -n <namespace>

$ sudo dnf install microshift-multus

$ sudo systemctl restart

$ oc get pods -n openshift-multus

NAME                READY   STATUS    RESTARTS   AGE
dhcp-daemon-dfbzw   1/1     Running   0          5h
multus-rz8xc        1/1     Running   0          5h

$ oc apply -f network-attachment-definition.yaml

apiVersion: "k8s.cni.cncf.io/v1"
kind: NetworkAttachmentDefinition
metadata:
  name: bridge-conf
spec:
  config: '{
      "cniVersion": "0.4.0",
      "type": "bridge",
      "bridge": "br-test",
      "mode": "bridge",
      "ipam": {
        "type": "host-local",
        "ranges": [
          [
            {
              "subnet": "10.10.0.0/24",
              "rangeStart": "10.10.0.20",
              "rangeEnd": "10.10.0.50",
              "gateway": "10.10.0.254"
             }
          ],
          [
            {
              "subnet": "fd00:IJKL:MNOP:10::0/64",
              "rangeStart": "fd00:IJKL:MNOP:10::1",
              "rangeEnd": "fd00:IJKL:MNOP:10::9"
        "dataDir": "/var/lib/cni/br-test"
      }
    }'

$ oc apply -f ./<test_bridge>.yaml

pod/test_bridge created

apiVersion: v1
kind: Pod
metadata:
  name: test_bridge
  annotations:
    k8s.v1.cni.cncf.io/networks: bridge-conf
  labels:
    app: test_bridge
spec:
  terminationGracePeriodSeconds: 0
  containers:
  - name: hello-microshift
    image: quay.io/microshift/busybox:1.36
    command: ["/bin/sh"]
    args: ["-c", "while true; do echo -ne \"HTTP/1.0 200 OK\r\nContent-Length: 16\r\n\r\nHello MicroShift\" | nc -l -p 8080 ; done"]
    ports:
    - containerPort: 8080
      protocol: TCP
    securityContext:
      allowPrivilegeEscalation: false
      capabilities:
        drop:
        - ALL
      runAsNonRoot: true
      runAsUser: 1001
      runAsGroup: 1001
      seccompProfile:
        type: RuntimeDefault

apiVersion: v1
kind: Pod
metadata:
  annotations:
    k8s.v1.cni.cncf.io/networks: bridge-conf
# ...

$ oc get pod <name> -o yaml

$ oc get pod <test_bridge> -o yaml

apiVersion: v1
kind: Pod
metadata:
  annotations:
    k8s.v1.cni.cncf.io/networks: bridge-conf
    k8s.v1.cni.cncf.io/network-status: |-
      [{
          "name": "ovn-kubernetes",
          "interface": "eth0",
          "ips": [
              "10.42.0.18"
          ],
          "default": true,
          "dns": {}
      },{
          "name": "bridge-conf",
          "interface": "net1",
          "ips": [
              "20.2.2.100"
          ],
          "mac": "22:2f:60:a5:f8:00",
          "dns": {}
      }]
  name: pod
  namespace: default
spec:
# ...
status:
# ...

$ oc get pod

NAME          READY   STATUS    RESTARTS   AGE
test_bridge   1/1     Running   0          81s

$ ip a show br-test

22: br-test: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether 96:bf:ca:be:1d:15 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::34e2:bbff:fed2:31f2/64 scope link
       valid_lft forever preferred_lft forever

$ sudo ip addr add 10.10.0.10/24 dev br-test

$ ip a show br-test

22: br-test: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether 96:bf:ca:be:1d:15 brd ff:ff:ff:ff:ff:ff
    inet 10.10.0.10/24 scope global br-test
       valid_lft forever preferred_lft forever
    inet6 fe80::34e2:bbff:fed2:31f2/64 scope link
       valid_lft forever preferred_lft forever

$ oc get pod test-bridge --output=jsonpath='{.metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}'

[{
    "name": "ovn-kubernetes",
    "interface": "eth0",
    "ips": [
        "10.42.0.17"
    ],
    "mac": "0a:58:0a:2a:00:11",
    "default": true,
    "dns": {}
},{
    "name": "default/bridge-conf",
    "interface": "net1",
    "ips": [
        "10.10.0.20"
    ],
    "mac": "82:01:98:e5:0c:b7",
    "dns": {}

$ oc exec -ti test-bridge -- ip a

1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0@if21: <BROADCAST,MULTICAST,UP,LOWER_UP,M-DOWN> mtu 1500 qdisc noqueue
    link/ether 0a:58:0a:2a:00:11 brd ff:ff:ff:ff:ff:ff
    inet 10.42.0.17/24 brd 10.42.0.255 scope global eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::858:aff:fe2a:11/64 scope link
       valid_lft forever preferred_lft forever
3: net1@if23: <BROADCAST,MULTICAST,UP,LOWER_UP,M-DOWN> mtu 1500 qdisc noqueue
    link/ether 82:01:98:e5:0c:b7 brd ff:ff:ff:ff:ff:ff
    inet 10.10.0.20/24 brd 10.10.0.255 scope global net1
       valid_lft forever preferred_lft forever
    inet6 fe80::8001:98ff:fee5:cb7/64 scope link
       valid_lft forever preferred_lft forever

$ curl 10.10.0.20:8080

Hello MicroShift

apiServer:
# ...
ingress:
  accessLogging:
    destination:
      container:
        maxLength: 1024
      syslog:
        address: ""
        facility: ""
        maxLength: 1024
        port: 0
      type: ""
    httpCaptureCookies:
      - matchType: ""
        maxLength: 0
        name: ""
        namePrefix: ""
    httpCaptureHeaders:
      request:
        - maxLength: 0
          name: ""
      response:
        - maxLength: 0
          name: ""
    httpLogFormat: ""
    status: Disabled
  certificateSecret: router-certs-custom
  clientTLS:
    allowedSubjectPatterns: []
    clientCA:
      name: ""
    clientCertificatePolicy: ""
  defaultHTTPVersion: 1
  forwardedHeaderPolicy: Append
  httpCompression:
    mimeTypes:
      - ""
  httpEmptyRequestsPolicy: Respond
  httpErrorCodePages:
      name: ""
  listenAddress: []
  logEmptyRequests: Log
  ports:
     http: 80
     https: 443
  routeAdmissionPolicy:
    namespaceOwnership: InterNamespaceAllowed
    wildcardPolicy: WildcardsDisallowed
  status: Managed
  tlsSecurityProfile:
    type:
    custom:
      ciphers:[]
      minTLSVersion:""
    intermediate: {}
    old: {}
  tuningOptions:
    clientFinTimeout: 1s
    clientTimeout: 30s
    headerBufferBytes: 0
    headerBufferMaxRewriteBytes: 0
    healthCheckInterval: 5s
    maxConnections: 0
    serverFinTimeout: 1s
    serverTimeout: 30s
    threadCount: 4
    tlsInspectDelay: 5s
    tunnelTimeout: 1h
# ...

$ sudo systemctl start microshift

$ sudo systemctl restart microshift

$ oc create secret tls <secret> \
     --cert=</path/to/cert.crt> \
     --key=</path/to/cert.key> \
     -n openshift-ingress

$ sudo systemctl start microshift

$ sudo systemctl restart microshift

 ...
spec:
  tlsSecurityProfile:
    type: Custom
    custom:
      ciphers:
      - ECDHE-ECDSA-CHACHA20-POLY1305
      - ECDHE-RSA-CHACHA20-POLY1305
      - ECDHE-RSA-AES128-GCM-SHA256
      - ECDHE-ECDSA-AES128-GCM-SHA256
      minTLSVersion: VersionTLS11
 ...

$ sudo systemctl restart microshift

# ...
ingress:
  ports:
    http: 80
    https: 443
  routeAdmissionPolicy:
    namespaceOwnership: InterNamespaceAllowed
  status: Removed
# ...

$ sudo systemctl restart microshift

# ...
ingress:
  ports:
    http: 80
    https: 443
  routeAdmissionPolicy:
    namespaceOwnership: InterNamespaceAllowed
  status: Managed
# ...

$ sudo systemctl restart microshift

# ...
ingress:
  listenAddress:
    - "<host_network>"
# ...

# ...
ingress:
  listenAddress:
    - 10.2.1.100
# ...

# ...
ingress:
  listenAddress:
    - 10.2.1.100
    - 10.2.2.10
    - ens3
# ...

$ sudo systemctl restart microshift

# ...
ingress:
  routeAdmissionPolicy:
    namespaceOwnership: Strict
# ...

$ sudo systemctl restart microshift

$ oc expose pod hello-microshift -n $namespace

$ oc expose svc/hello-microshift --hostname=microshift.com $namespace

$ oc annotate route <route_name> router.openshift.io/cookie_name="<cookie_name>"

$ ROUTE_NAME=$(oc get route <route_name> -o jsonpath='{.spec.host}')

$ curl $ROUTE_NAME -k -c /tmp/cookie_jar

$ curl $ROUTE_NAME -k -b /tmp/cookie_jar

apiVersion: route.openshift.io/v1
kind: Route
# ...
spec:
  host: app.example.com
  tls:
    termination: edge
  to:
    kind: Service
    name: app-example
  httpHeaders:
    actions:
      response:
      - name: Content-Location
        action:
          type: Set
          set:
            value: /lang/en-us
# ...

$ oc -n app-example create -f app-example-route.yaml

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: frontend
  annotations:
    route.openshift.io/termination: "reencrypt"
    route.openshift.io/destination-ca-certificate-secret: secret-ca-cert
spec:
  rules:
  - host: www.example.com
    http:
      paths:
      - backend:
          service:
            name: frontend
            port:
              number: 443
        path: /
        pathType: Prefix
  tls:
  - hosts:
    - www.example.com
    secretName: example-com-tls-certificate
# ...

$ oc get routes

NAME             HOST/PORT         PATH    SERVICES    PORT    TERMINATION          WILDCARD
frontend-gnztq   www.example.com           frontend    443     reencrypt/Redirect   None

apiVersion: route.openshift.io/v1
kind: Route
metadata:
  name: frontend-gnztq
  ownerReferences:
  - apiVersion: networking.k8s.io/v1
    controller: true
    kind: Ingress
    name: frontend
    uid: 4e6c59cc-704d-4f44-b390-617d879033b6
spec:
  host: www.example.com
  path: /
  port:
    targetPort: https
  tls:
    certificate: |
      -----BEGIN CERTIFICATE-----
      [...]
      -----END CERTIFICATE-----
    insecureEdgeTerminationPolicy: Redirect
    key: |
      -----BEGIN RSA PRIVATE KEY-----
      [...]
      -----END RSA PRIVATE KEY-----
    termination: reencrypt
    destinationCACertificate: |
      -----BEGIN CERTIFICATE-----
      [...]
      -----END CERTIFICATE-----
  to:
    kind: Service
    name: frontend

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: frontend
  ...
spec:
  rules:
    ...
  tls:
  - {}

$ oc create -f example-ingress.yaml

$ oc create secret generic dest-ca-cert --from-file=tls.crt=<file_path>

$ oc -n test-ns create secret generic dest-ca-cert --from-file=tls.crt=tls.crt

secret/dest-ca-cert created

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: frontend
  annotations:
    route.openshift.io/termination: "reencrypt"
    route.openshift.io/destination-ca-certificate-secret: secret-ca-cert
...

apiServer:
  namedCertificates:
   - certPath: ~/certs/api_fqdn_1.crt
     keyPath:  ~/certs/api_fqdn_1.key
   - certPath: ~/certs/api_fqdn_2.crt
     keyPath:  ~/certs/api_fqdn_2.key
     names:
     - api_fqdn_1
     - *.apps.external.com

$ systemctl microshift restart

$ journalctl -u microshift -b0 | grep tls-sni-cert-key

Jan 24 14:53:00 localhost.localdomain microshift[45313]: kube-apiserver I0124 14:53:00.649099   45313 flags.go:64] FLAG: --tls-sni-cert-key="[/home/eslutsky/dev/certs/server.crt,/home/eslutsky/dev/certs/server.key;/var/lib/microshift/certs/kube-apiserver-external-signer/kube-external-serving/server.crt,/var/lib/microshift/certs/kube-apiserver-external-signer/kube-external-serving/server.key;/var/lib/microshift/certs/kube-apiserver-localhost-signer/kube-apiserver-localhost-serving/server.crt,/var/lib/microshift/certs/kube-apiserver-localhost-signer/kube-apiserver-localhost-serving/server.key;/var/lib/microshift/certs/kube-apiserver-service-network-signer/kube-apiserver-service-network-serving/server.crt,/var/lib/microshift/certs/kube-apiserver-service-network-signer/kube-apiserver-service-network-serving/server.key

$ openssl s_client -connect <SNI_ADDRESS>:6443 -showcerts | openssl x509 -text -noout -in - | grep -C 1 "Alternative\|CN"

$ sudo microshift-cleanup-data --cert

Stopping MicroShift services
Removing MicroShift certificates
MicroShift service was stopped
Cleanup succeeded

$ sudo systemctl start microshift