Date: 2025-05-12 23:31
Tags: # 
Related Ideas: [[]]
Links: https://febryandana.medium.com/deploy-ceph-openstack-cluster-is-easy-2e907f2e8970 http://www.yet.org/2014/09/openvswitch-troubleshooting/ https://gist.github.com/kriegalex/3c4fd2493c19dac8388b737c47157e92 https://docs.openstack.org/kolla-ansible/latest/admin/production-architecture-guide.html https://hackmd.io/@yujungcheng/ByXxq-_Vj https://blog.rhysgoodwin.com/it/openstack-ceph-hyperconverged/ https://docs.openstack.org/kolla-ansible/2024.2/user/adding-and-removing-hosts.html https://docs.openstack.org/cinder/2024.2/configuration/block-storage/backup-drivers.html https://bugs.launchpad.net/nova/+bug/1195361 https://bugzilla.redhat.com/show_bug.cgi?id=1430264&utm_source=chatgpt.com https://docs.ceph.com/en/latest/rbd/rbd-openstack/ https://infohub.delltechnologies.com/en-us/l/architecture-guide-ready-architecture-for-red-hat-openstack-platform-version-16-1-flex-2/solution-with-25gbe-100gbe-networking-overview/


```sh
kolla-ansible  bootstrap-servers  -i multinode --configdir $PWD --passwords passwords.yml --limit compute02
kolla-ansible  pull  -i inventory --configdir $PWD --passwords passwords.yml --limit compute02
kolla-ansible  deploy  -i inventory --configdir $PWD --passwords passwords.yml --limit compute03

```

## ceph

```
Ceph Dashboard is now available at:

             URL: https://8443/
            User: admin
        Password: myut32kv19

```

```sh
apt install -y cephadm
cephadm install

cephadm bootstrap --mon-ip 10.36.0.31 --cluster-network 10.37.0.0/24
# удаление всех подписей файловых систем
wipefs -a /dev/sdb  
# или полная очистка GPT
sgdisk --zap-all /dev/sdb  


ceph orch host ls


 ceph orch host add ru-tul-dc01-storage02 10.36.0.32
 ceph orch apply mon ru-tul-dc01-storage02
	  ceph orch apply mgr ru-tul-dc01-storage02
  ceph orch daemon add osd ru-tul-dc01-storage02:/dev/sdb

echo ""

root@ru-tul-dc01-storage01:~# ceph auth get-or-create client.glance mon 'profile rbd' osd 'profile rbd pool=images' mgr 'profile rbd pool=images'
[client.glance]
        key = AQCeR0honq8YAxAA590U3GOUiiKiYDfSdGNWCQ==
root@ru-tul-dc01-storage01:~#



root@ru-tul-dc01-storage01:~# ceph auth get-or-create client.cinder mon 'profile rbd' osd 'profile rbd pool=volumes, profile rbd pool=vms, profile rbd-read-only pool=images' mgr 'profile rbd pool=volumes, profile rbd pool=vms'
[client.cinder]
        key = AQCyR0ho/OM9EBAA7tYupWGCuwl2wDEg8OLDVw==




ceph osd pool create backups
rbd pool init backups
ceph auth get-or-create client.cinder-backup mon 'profile rbd' osd 'profile rbd pool=backups' mgr 'profile rbd pool=backups'
[client.cinder-backup]
        key = AQC8+01o55a4KBAAXNSN1hJBlJIHP6eiV15OsA==


```

2:29

```
cp /usr/share/kolla-ansible/ansible/roles/neutron/templates/ml2_conf.ini.j2 /etc/kolla/config/neutron/ml2_conf.ini

export OS_CACERT=/home/ubuntu/kolla-ztk/certificates/ca/root.crt


ovs-vsctl get open . external_ids:ovn-bridge-mappings

kolla-ansible  destroy  -i multinode --configdir $PWD --passwords passwords.yml --limit compute02 --yes-i-really-really-mean-it

openstack  server create     --image cirros     --flavor m1.tiny     --key-name mykey     --network demo-net     demo2


sudo rm -f /var/run/libvirt/libvirt-sock*

enp2s2
```

```
kolla_internal_vip_address: "10.32.0.100"

# External (public/Horizon/Floating IP)
kolla_external_vip_interface: "ens18"
kolla_external_vip_address:   "172.20.71.71"

#####################
## Network Ifaces  ##
#####################
# Management / Private API
network_interface:          "ens19"
api_interface:              "ens20"

# External public network
neutron_external_interface: "ens18"
public_api_interface:       "ens18"

# Tenant overlay
tunnel_interface:           "ens23"



sudo rm -f /var/run/libvirt/libvirt-sock*

```

| Network name<br>          | IP Pool        |         |
| ------------------------- | -------------- | ------- |
| Management                | 10.31.0.1/24   | vmbr31  |
| Overlay and tenant        | 10.35.0.1/24   | vmbr35  |
| External                  | 172.20.70.1/23 | vlan400 |
| Storage                   | 10.36.0.1/24   | vmbr36  |
| Private API               | 10.32.0.1/24   | vmbr32  |
| Public API                | 172.20.70.1/23 | vlan400 |
| Provisionig               | 10.33.0.1/24   | vmbr33  |
| Internal Network          | 10.34.0.1/24   | vmbr34  |
| openstack-ceph-clustering | 10.37.0.1/24   | vmbr37  |


infra01

ens18 The external interface/public api 172.20.70.10/23  (необязательно если управление через idrac)
ens19 The management interface vmbr31 10.31.0.10/24
ens 20 Private API 10.32.0.10/24
ens 21 Private API 10.32.0.10/24


control01-03

ens18 The external interface 172.20.70.11-13/23 (необязательно если управление через idrac)
ens19 The management interface 10.31.0.11.-13/24
ens 20 Private API 10.32.0.11-13/24
ens 21 Provisionig 10.33.0.11-13/24
ens 22  Internal Network 10.34.0.11-13/24
ens 23 Overlay and tenant 10.35.0.11-13/24
enp2s1 Storage 10.36.0.11-13/24
enp2s2 The external interface

compute01-03

ens18 The external interface 172.20.70.21-23/23 (необязательно если управление через idrac)
ens19 The management interface  10.31.0.21-23/24
ens 20 Private API 10.32.0.21-23/24
ens 21 Provisionig 10.33.0.21-23/24
ens 22  Internal Network 10.34.0.21-23/24
ens 23 Overlay and tenant 10.35.0.21-23/24
ens 24 Storage 10.36.0.21-23/24
enp2s2 The external interface


storage01-03

ens18 The external interface 172.20.70.31-33/23
ens19 The management interface  10.31.0.31-33/24
ens 20 Private API 10.32.0.31-33/24
ens 21 Provisionig 10.33.0.31-33/24
ens22 Storage 10.36.0.31-33/24
ens23  Storage Cluster 10.37.0.31-33/24

```sh
---

# workaround
workaround_ansible_issue_8743: "yes"

#####################
## General         ##
#####################
kolla_base_distro:      "ubuntu"
kolla_install_type:     "source"
#enable_neutron_provider_networks: "yes"
neutron_plugin_agent: "ovn"
enable_openstack_core: "yes"
neutron_ovn_distributed_fip: "yes"
#####################
## VIP Addresses   ##
#####################
# Private API VIP (mgmt/internal)
kolla_internal_vip_address: "10.32.0.100"

# External (public/Horizon/Floating IP)
kolla_external_vip_interface: "ens18"
kolla_external_vip_address:   ""

#####################
## Network Ifaces  ##
#####################
# Management / Private API
network_interface:          "ens19"
api_interface:              "ens20"

# External public network
neutron_external_interface: "enp2s2"
#public_api_interface:       "ens20"

# Tenant overlay
tunnel_interface:           "ens23"

#####################
## HA / Proxy      ##
#####################
enable_haproxy:             "yes"
keepalived_virtual_router_id: "51"

#####################
## Cinder/LVM      ##
#####################
enable_cinder:              "yes"
enable_cinder_backend_lvm:  "yes"
cinder_volume_group:        "cinder-volumes"
enable_cinder_backup:       "yes"
enable_cinder_backend_iscsi: "yes"
enable_iscsid:              "yes"

#####################
## Disable extras  ##
#####################
enable_ceph:    "no"
enable_manila:  "no"
enable_gnocchi: "no"

#####################
## TLS / SSL       ##
#####################
kolla_enable_tls_external:  "yes"
kolla_enable_tls_internal:  "yes"
kolla_enable_tls_backend:    "yes"
kolla_copy_ca_into_containers: "yes"
openstack_cacert:           "/etc/ssl/certs/ca-certificates.crt"

#####################
## Monitoring      ##
#####################
enable_prometheus:                "yes"
enable_prometheus_openstack_exporter: "yes"
enable_grafana:                   "yes"
enable_prometheus_node_exporter:  "yes"

```

```sh
pvcreate /dev/sdb1
vgcreate cinder-volumes /dev/sdb1
vgs

sudo apt update
sudo apt install python3-docker git python3-dev libffi-dev gcc libssl-dev  -y
sudo apt install build-essential libdbus-glib-1-dev libgirepository1.0-dev -y


sudo apt install python3-venv -y

python3 -m venv openstack-caracal
source openstack-caracal/bin/activate

pip install -U pip  
pip install docker
pip install 'ansible-core>=2.15,<2.17'

pip install 'kolla-ansible==19.4.0'

mkdir all-in-one/  
cp -r openstack-caracal/share/kolla-ansible/etc_examples/kolla/*  all-in-one/
cp openstack-caracal/share/kolla-ansible/ansible/inventory/* all-in-one/
kolla-ansible install-deps
cd all-in-one
kolla-genpwd -p $PWD/passwords.yml





kolla-ansible  certificates -i multinode --configdir $PWD --passwords passwords.yml 
kolla-ansible  bootstrap-servers  -i multinode --configdir $PWD --passwords passwords.yml
kolla-ansible  genconfig -i multinode --configdir $PWD --passwords passwords.yml 
kolla-ansible  prechecks  -i multinode --configdir $PWD --passwords passwords.yml
kolla-ansible  pull  -i multinode --configdir $PWD --passwords passwords.yml
kolla-ansible  deploy  -i multinode --configdir $PWD --passwords passwords.yml
kolla-ansible  post-deploy  -i multinode --configdir $PWD --passwords passwords.yml









openstack network create  --share --external \
  --provider-physical-network physnet1 \
  --provider-network-type flat provider


openstack subnet create --network physnet1   --allocation-pool start=,end=   --dns-nameserver --gateway    --subnet-range   provider

``` 



### 3.1. ru-tul-dc01-infranode01

Infra-нода (Director/DevStack), **управление** идёт с неё, **не подключена** к External и Tenant-Tunnel:


```sh
# This file is generated from information provided by the datasource.  Changes
# to it will not persist across an instance reboot.  To disable cloud-init's
# network configuration capabilities, write a file
# /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following:
# network: {config: disabled}
network:
    version: 2
    ethernets:
        eth0:
            addresses:
            - 172.20.71.58/23
            dhcp6: true
            match:
                macaddress: bc:24:11:83:0b:0c
            nameservers:
                addresses:
                - 10.77.12.47
                - 10.77.12.87
                search:
                - 
            routes:
            -   to: default
                via: 172.20.70.1
            set-name: eth0
        eth1:
            addresses:
            - 10.31.0.10/24
            dhcp6: true
            match:
                macaddress: bc:24:11:75:bf:d0
            nameservers:
                addresses:
                - 10.77.12.47
                - 10.77.12.87
                search:
                - 
            set-name: eth1
        eth2:
            addresses:
            - 10.32.0.10/24
            dhcp6: true
            match:
                macaddress: bc:24:11:97:eb:44
            nameservers:
                addresses:
                - 172.20.71.47
                - 172.20.71.87
                search:
                - 
            set-name: eth2
        ens21:
            addresses:
            - 10.33.0.10/24
            dhcp6: true
```

### 3.2. ru-tul-dc01-control0[1–3]
```
network:
  version: 2
  renderer: networkd

  ethernets:
    # 1) Физический интерфейс ens18 — здесь привязан внешний статический IP
    ens18:
      dhcp4: false
      dhcp6: false
      addresses:
        - 172.20.71.59/23    #The external interface
      nameservers:
        addresses:
          - 172.20.71.47
          - 172.20.71.87
        search:
          - 
      routes:
        - to: default
          via: 172.20.70.1
    ens19:
      dhcp4: false
      addresses:
        - 10.31.0.11/24      #The management interface
    ens20:
      dhcp4: false
      addresses:
        - 10.32.0.11/24      # Private API
    ens21:
      dhcp4: false
      addresses:
        - 10.33.0.11/24      # Provisionig
    ens22:
      dhcp4: false
      addresses:
        - 10.34.0.11/24      # Internal Network
    ens23:
      dhcp4: false
      addresses:
        - 10.35.0.11/24      # Overlay and tenant
    enp2s1:
      dhcp4: false
      addresses:
        - 10.36.0.11/24      # Storage
    enp2s2:
      dhcp4: false   #neutron_external_interface



```

### 3.3. ru-tul-dc01-compute0[1–3]



```
network:
  version: 2
  renderer: networkd

  ethernets:
    # 1) Физический интерфейс ens18 — здесь привязан внешний статический IP
    ens18:
      dhcp4: false
      dhcp6: false
      addresses:
        - 172.20.71.73/23   # The external interface
      nameservers:
        addresses:
          - 172.20.71.47
          - 172.20.71.87
        search:
          - 
      routes:
        - to: default
          via: 172.20.70.1
    ens19:
      dhcp4: false
      addresses:
        - 10.31.0.31/24      # The management interface

    ens20:
      dhcp4: false
      addresses:
        - 10.32.0.31/24      # Private API

    ens21:
      dhcp4: false
      addresses:
        - 10.33.0.31/24      #  Provisionig

    ens22:
      dhcp4: false
      addresses:
        - 10.36.0.31/24      # storage
    ens23:
      dhcp4: false
      addresses:
        - 10.37.0.31/24      # Storage Cluster



```
### 3.4. ru-tul-dc01-storage01

Storage-нода (Ceph), **только** сети хранения:


```
network:
  version: 2
  renderer: networkd

  ethernets:
    # 1) Физический интерфейс ens18 — здесь привязан внешний статический IP
    ens18:
      dhcp4: false
      dhcp6: false
      addresses:
        - 172.20.71.74/23   # The external interface
      nameservers:
        addresses:
          - 172.20.71.47
          - 172.20.71.87
        search:
          - 
      routes:
        - to: default
          via: 172.20.70.1
    ens19:
      dhcp4: false
      addresses:
        - 10.31.0.32/24      # The management interface

    ens20:
      dhcp4: false
      addresses:
        - 10.32.0.32/24      # Private API

    ens21:
      dhcp4: false
      addresses:
        - 10.33.0.32/24      #  Provisionig

    ens22:
      dhcp4: false
      addresses:
        - 10.36.0.32/24      # storage
      mtu: 9000

    ens23:
      dhcp4: false
      addresses:
        - 10.37.0.32/24      # Storage Cluster
      mtu: 9000


```