How to Deploy an OpenSearch Cluster on AWS

This assumes that you have already deployed an EKS cluster.

Set your variables

export ACCOUNT_ID="MYACCOUNT"
export AWS_REGION="us-west-2"
export CLUSTER="CLUSTERNAME"
export ES_DOMAIN_NAME="dev-eks-c1-log"

Set your provider

eksctl utils associate-iam-oidc-provider \
    --cluster $CLUSTER \
    --approve

Create a directory for our configuration files

mkdir ./logging/

Create our AWS IAM policy definition

cat <<EoF > ./logging/fluent-bit-policy.json
{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Action": [
                "es:ESHttp*"
            ],
            "Resource": "arn:aws:es:${AWS_REGION}:${ACCOUNT_ID}:domain/${ES_DOMAIN_NAME}",
            "Effect": "Allow"
        }
    ]
}
EoF

Create our policy in AWS IAM

aws iam create-policy   \
  --policy-name fluent-bit-policy \
  --policy-document file://./logging/fluent-bit-policy.json

kubectl create namespace logging

eksctl create iamserviceaccount \
    --name fluent-bit \
    --namespace logging \
    --cluster development-eks-cluster1 \
    --attach-policy-arn "arn:aws:iam::${ACCOUNT_ID}:policy/fluent-bit-policy" \
    --approve \
    --override-existing-serviceaccounts

kubectl -n logging describe sa fluent-bit

Set OpenSearch values

# name of our Amazon OpenSearch cluster
export ES_DOMAIN_NAME="eks-cluster-logs"

# Elasticsearch version
export ES_VERSION="OpenSearch_1.0"

# OpenSearch Dashboards admin user
export ES_DOMAIN_USER="admin"

# OpenSearch Dashboards admin password
export ES_DOMAIN_PASSWORD="$(openssl rand -base64 12)_Ek1$"

# Download and update the template using the variables created previously
curl -sS https://www.eksworkshop.com/intermediate/230_logging/deploy.files/es_domain.json \
  | envsubst > ./logging/es_domain.json

# Create the cluster
aws opensearch create-domain \
  --cli-input-json  file://./logging/es_domain.json

# wait until false
aws opensearch describe-domain --domain-name ${ES_DOMAIN_NAME} --query 'DomainStatus.Processing'

# We need to retrieve the Fluent Bit Role ARN
export FLUENTBIT_ROLE=$(eksctl get iamserviceaccount --cluster ${CLUSTER} --namespace logging -o json | jq '.[].status.roleARN' -r)

# Get the Amazon OpenSearch Endpoint
export ES_ENDPOINT=$(aws opensearch describe-domain --domain-name ${ES_DOMAIN_NAME} --output text --query "DomainStatus.Endpoint")

# Update the Elasticsearch internal database
curl -sS -u "${ES_DOMAIN_USER}:${ES_DOMAIN_PASSWORD}" \
    -X PATCH \
    https://${ES_ENDPOINT}/_opendistro/_security/api/rolesmapping/all_access?pretty \
    -H 'Content-Type: application/json' \
    -d'
[
  {
    "op": "add", "path": "/backend_roles", "value": ["'${FLUENTBIT_ROLE}'"]}
]
'

# Deploy Fluentbit

# get the Amazon OpenSearch Endpoint
export ES_ENDPOINT=$(aws es describe-elasticsearch-domain --domain-name ${ES_DOMAIN_NAME} --output text --query "DomainStatus.Endpoint")

# Set ES_ENDPOINT in the file

curl -Ss https://www.eksworkshop.com/intermediate/230_logging/deploy.files/fluentbit.yaml \
    | envsubst > ./logging/fluentbit.yaml

kubectl apply -f ./logging/fluentbit.yaml