.gitlab-ci.yml

include:
    # Auto devop
    - template: Auto-DevOps.gitlab-ci.yml

variables:
    GIT_SUBMODULE_STRATEGY: recursive
    FLAP_DEBUG: "true"
    FLAP_DIR: $CI_BUILDS_DIR/flap_dir
    FLAP_DATA: $CI_BUILDS_DIR/flap_data
    FLAP_CURRENT_VERSION: $CI_BUILDS_DIR/flap-box/flap

stages:
    - build
    - setup
    - test
    - deploy # dummy stage to follow the template guidelines
    - review
    - run
    - dast
    - staging
    - canary
    - production
    - incremental rollout 10%
    - incremental rollout 25%
    - incremental rollout 50%
    - incremental rollout 100%
    - performance
    - cleanup

test_script_executable:
    image: debian
    stage: test
    needs: []
    rules:
        - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
        - if: '$CI_COMMIT_BRANCH == "master"'
        - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+/'
    script:
        - find . -path './**/node_modules' -prune -o  -name '*.sh' ! -executable -print
        - test $(find . -path './**/node_modules' -prune -o  -name '*.sh' ! -executable -print | wc -l) == "0"

shellcheck:
    image: koalaman/shellcheck-alpine:latest
    stage: test
    needs: []
    rules:
        - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
        - if: '$CI_COMMIT_BRANCH == "master"'
        - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+/'
    before_script:
        - shellcheck --version
    script:
        - apk add git
        - git ls-files '*.sh' -z | xargs -r0 shellcheck --external-sources --shell=bash

test_dockerfile_sanity:
    image:
        name: instrumenta/conftest:latest
        entrypoint: [""]
    stage: test
    needs: []
    rules:
        - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
        - if: '$CI_COMMIT_BRANCH == "master"'
        - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+/'
    before_script:
        - conftest --version
    script:
        - find ./ -name Dockerfile | xargs -I {} conftest test --policy dockerfile-security.rego {}

build_docker_image:
    stage: build
    image: docker:stable
    services: ["docker:dind"]
    script:
        - DOCKER_DRIVER=overlay2
        - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
        - >
            docker build \
                --tag $CI_REGISTRY_IMAGE/${CI_COMMIT_REF_SLUG}:${CI_COMMIT_SHA} \
                --build-arg CI_COMMIT_REF_NAME=$CI_COMMIT_REF_NAME \
                ./
        - docker push $CI_REGISTRY_IMAGE/${CI_COMMIT_REF_SLUG}:${CI_COMMIT_SHA}

.start_flap:
    image: docker:stable
    parallel:
        matrix:
            - SERVICE_BUNDLE_NAME: office
              ENABLE_NEXTCLOUD: "true"
              ENABLE_COLLABORA: "true"
              ENABLE_SOGO: "true"
              ENABLE_MATRIX: "true"
              ENABLE_JITSI: "false"
              ENABLE_MONITORING: "true"
            - SERVICE_BUNDLE_NAME: social
              ENABLE_PEERTUBE: "true"
              ENABLE_FUNKWHALE: "true"
              ENABLE_MAILMAN: "true"
              ENABLE_MONITORING: "true"
              ENABLE_MATOMO: "true"
    services:
        - name: docker:dind
          # We use flap.test instead of flap.localhost because chrome won't resolve *.localhost on another IP.
          alias: flap.local flap.test auth.flap.test home.flap.test files.flap.test mail.flap.test matrix.flap.test chat.flap.test coturn.flap.test jitsi.flap.test weblate.flap.test analytics.flap.test video.flap.test monitoring.flap.test music.flap.test lists.flap.test office.flap.test
    needs: [build_docker_image]
    before_script:
        # Get DinD's ip.
        - FLAP_IP=$(cat /etc/hosts | grep flap.local | cut -f1)

        # Install git so it can be used in FLAP_IMAGE_TAG scripts.
        - apk add git patch

        # Share the docker container network with this container so it can talk to itself.
        #  ==> the network ports are being serve from the DinD container, that can be reach from the docker container but not from this container if we do not share the host network stack.
        # Add entry to the /etc/hosts file to resolve flap.local and *.flap.test
        - echo "Starting $(eval $FLAP_IMAGE_TAG)"
        - >
            docker run \
                --detach \
            \
                --name flap \
            \
                --network host \
                --add-host="flap.local:$FLAP_IP" \
                --add-host="flap.test:$FLAP_IP" \
                --add-host="auth.flap.test:$FLAP_IP" \
                --add-host="files.flap.test:$FLAP_IP" \
                --add-host="mail.flap.test:$FLAP_IP" \
                --add-host="matrix.flap.test:$FLAP_IP" \
                --add-host="chat.flap.test:$FLAP_IP" \
                --add-host="jitsi.flap.test:$FLAP_IP" \
                --add-host="weblate.flap.test:$FLAP_IP" \
                --add-host="analytics.flap.test:$FLAP_IP" \
                --add-host="video.flap.test:$FLAP_IP" \
                --add-host="monitoring.flap.test:$FLAP_IP" \
                --add-host="music.flap.test:$FLAP_IP" \
                --add-host="lists.flap.test:$FLAP_IP" \
                --add-host="office.flap.test:$FLAP_IP" \
            \
                --env LOG_DRIVER=json-file \
            \
                --volume /var/run/docker.sock:/var/run/docker.sock \
            \
                --volume $CI_BUILDS_DIR:$CI_BUILDS_DIR \
                --volume /etc/letsencrypt/live/flap:/etc/letsencrypt/live/flap \
            \
                --env FLAP_DIR=$FLAP_DIR \
                --env FLAP_DATA=$FLAP_DATA \
                --env FLAP_CURRENT_VERSION=$FLAP_CURRENT_VERSION \
                --env COMPOSE_HTTP_TIMEOUT=3000 \
                --env CI_JOB_NAME="$CI_JOB_NAME" \
                --env FLAP_DEBUG=$FLAP_DEBUG \
            \
                --env ENABLE_NEXTCLOUD=$ENABLE_NEXTCLOUD \
                --env ENABLE_COLLABORA=$ENABLE_COLLABORA \
                --env ENABLE_SOGO=$ENABLE_SOGO \
                --env ENABLE_MATRIX=$ENABLE_MATRIX \
                --env ENABLE_JITSI=$ENABLE_JITSI \
                --env ENABLE_PEERTUBE=$ENABLE_PEERTUBE \
                --env ENABLE_FUNKWHALE=$ENABLE_FUNKWHALE \
                --env ENABLE_MAILMAN=$ENABLE_MAILMAN \
                --env ENABLE_MONITORING=$ENABLE_MONITORING \
                --env ENABLE_MATOMO=$ENABLE_MATOMO \
            \
                --workdir $FLAP_DIR \
                $(eval $FLAP_IMAGE_TAG) \
                /bin/sh -c "while true; do sleep 1000; done"

        # Copy the content of flap:/opt/flap into flap:$FLAP_DIR.
        # This is necessary to be able to bind volumes to containers started from the flap container.
        # https://stackoverflow.com/questions/46437147/gitlab-ci-docker-in-docker-cant-create-volume
        - docker exec flap cp -rT /opt/flap $FLAP_DIR
        # Update the flapctl link so it maps to the new location.
        - docker exec flap ln -sf $FLAP_DIR/system/cli/flapctl.sh /bin/flapctl

        # Copy flapctl.env for pipeline.
        - mkdir --parents $FLAP_DATA/system
        - if [ ! -f flap.$SERVICE_BUNDLE_NAME.tar.gz ]; then cp $FLAP_DIR/system/flapctl.examples.d/pipeline.env $FLAP_DATA/system/flapctl.env; fi

        # Patch current version.
        # This is useful to make CI passe as sometime FLAP versions stop working because of external dependencies.
        - find $FLAP_CURRENT_VERSION/system/patches/ -name "$(eval $BASE_VERSION)"'.*.patch' -exec patch --dir $FLAP_DIR -p1 -i {} \;

        # Restore flap_data if it exists.
        - if [ -f flap.$SERVICE_BUNDLE_NAME.tar.gz ]; then cp $FLAP_DIR/system/flapctl.examples.d/pipeline.restore.env $FLAP_DATA/system/flapctl.env; fi
        - apk add tar
        - if [ -f flap.$SERVICE_BUNDLE_NAME.tar.gz ]; then tar -xzf flap.$SERVICE_BUNDLE_NAME.tar.gz -C $CI_BUILDS_DIR/..; fi
        - if [ -f flap.$SERVICE_BUNDLE_NAME.tar.gz ]; then docker exec flap flapctl backup restore; fi

        - docker exec flap flapctl config show
        - docker exec flap flapctl start

        - if [ -f flap.$SERVICE_BUNDLE_NAME.tar.gz ]; then rm -rf flap.*.tar.gz; fi

# Run start_flap and CLI unit tests
test_cli:
    stage: test
    extends: .start_flap
    rules:
        - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
        - if: '$CI_COMMIT_BRANCH == "master"'
        - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+/'
    needs: [build_docker_image]
    variables:
        FLAP_IMAGE_TAG: echo $CI_REGISTRY_IMAGE/${CI_COMMIT_REF_SLUG}:${CI_COMMIT_SHA}
    script:
        - docker exec flap flapctl test setup tls

setup_current:
    stage: setup
    extends: .start_flap
    retry: 1
    rules:
        - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
        - if: '$CI_COMMIT_BRANCH == "master"'
        - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+/'
    variables:
        FLAP_IMAGE_TAG: echo $CI_REGISTRY_IMAGE/${CI_COMMIT_REF_SLUG}:${CI_COMMIT_SHA}
    cache:
        key: flap_data_$SERVICE_BUNDLE_NAME
        paths: [flap.$SERVICE_BUNDLE_NAME.tar.gz]
        policy: push
    script:
        - docker exec flap flapctl users create_admin
        - docker exec flap flapctl domains generate_local flap.test

        # Backup $FLAP_DATA
        - docker exec flap flapctl backup
        - tar -czf flap.$SERVICE_BUNDLE_NAME.tar.gz $CI_BUILDS_DIR/flap_backup

.setup_from:
    stage: setup
    extends: .start_flap
    retry: 1
    rules:
        - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
        - if: '$CI_COMMIT_BRANCH == "master"'
        - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+/'
    needs: []
    variables:
        FLAP_IMAGE_TAG: echo $CI_REGISTRY_IMAGE:$(eval $BASE_VERSION)
    cache:
        key: flap_data_$SERVICE_BUNDLE_NAME
        paths: [flap.$SERVICE_BUNDLE_NAME.tar.gz]
        policy: push
    script:
        - docker exec flap flapctl users create_admin
        - docker exec flap flapctl domains generate_local flap.test

        # TODO: Remove when tested versions checkout tag during builds
        - docker exec flap git checkout "$(eval $BASE_VERSION)"
        - docker exec flap flapctl update

        # Update to the current commit if we are on a branch or on a merge request.
        # TODO: The condition is broken, it is always true, but the update do not work as the arg is empty.
        - if [ "${CI_COMMIT_BRANCH:-} != ''" ]; then docker exec flap flapctl update $CI_COMMIT_BRANCH; fi
        - if [ "$CI_PIPELINE_SOURCE" = "merge_request_event" ]; then docker exec flap flapctl update $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA; fi

        - test "$(git status --porcelain)" == ""
        - test "$(docker exec flap flapctl migrate status)" == ""

        # Backup $FLAP_DATA
        - docker exec flap flapctl backup
        - tar -czf flap.$SERVICE_BUNDLE_NAME.tar.gz $CI_BUILDS_DIR/flap_backup

setup_from_1:
    extends: .setup_from
    variables:
        BASE_VERSION: git tag --sort version:refname | tail -1 | head -1

setup_from_2:
    extends: .setup_from
    variables:
        BASE_VERSION: git tag --sort version:refname | tail -2 | head -1

setup_from_3:
    extends: .setup_from
    variables:
        BASE_VERSION: git tag --sort version:refname | tail -3 | head -1

.e2e:
    stage: test
    retry: 1
    extends: .start_flap
    rules:
        - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
        - if: '$CI_COMMIT_BRANCH == "master"'
        - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+/'
    cache:
        key: flap_data_$SERVICE_BUNDLE_NAME
        paths: [flap.$SERVICE_BUNDLE_NAME.tar.gz]
        policy: pull
    variables:
        SCREEN_WIDTH: "1920"
        SCREEN_HEIGHT: "1080"
        FLAP_IMAGE_TAG: echo $CI_REGISTRY_IMAGE/${CI_COMMIT_REF_SLUG}:${CI_COMMIT_SHA}
        # Don't download chromium during puppeteer installation.
        PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: "true"
    artifacts:
        when: always
        reports:
            junit: system/e2e/output/result.xml
        paths:
            - system/e2e/output
    script:
        # Install chromium: https://github.com/puppeteer/puppeteer/blob/master/docs/troubleshooting.md#running-on-alpine
        - >
            apk add --no-cache --update \
              chromium \
              nss \
              freetype \
              freetype-dev \
              harfbuzz \
              ca-certificates \
              ttf-freefont \
              bash \
              nodejs \
              npm \
              python3 \
              make

        - bash ./system/scripts/run_e2e_tests.sh

e2e_current:
    extends: .e2e
    needs: [setup_current]

e2e_from_1:
    extends: .e2e
    needs: [setup_from_1]

e2e_from_2:
    extends: .e2e
    needs: [setup_from_2]

e2e_from_3:
    extends: .e2e
    needs: [setup_from_3]

build_disk_image:
    stage: build
    image: ubuntu:18.04
    rules:
        - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.\d+/'
          when: manual
    allow_failure: true
    artifacts:
        name: "$CI_COMMIT_REF_NAME"
    script:
        # Setup ssh.
        - "which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )"
        - eval $(ssh-agent -s)
        - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
        - mkdir -p ~/.ssh
        - chmod 700 ~/.ssh
        - echo "$SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
        - chmod 644 ~/.ssh/known_hosts

        # Setting up armbian_build folder on the remote server if needed.
        - ssh root@flap.id 'git clone --depth 1 https://github.com/armbian/build /mnt/volume1/armbian_build || true'

        # Customize the userpatch install_flap script to use the current tag.
        - sed -i "s/\$VERSION/${CI_COMMIT_REF_NAME/\//\\\/}/g" ./system/img_build/userpatches/customize-image.sh
        - scp -r ./system/img_build/userpatches root@flap.id:/mnt/volume1/armbian_build/

        # Download currently used docker images so they can be loaded in the final image.
        # This is a long one-liner, sorry.
        # - It gets the image names in each docker-compose.yml.
        # - Then it ask docker to pull the image using the experimental flag "--platform"
        # - It finally use the `docker save` cmd to extract and zip the image to a final 'userpatches/overlay/{image_name}.tar.gz' file.
        # The final pull and extract is done on the remote server.
        - apt install -y python3 python3-pip jq
        - pip3 install yq
        - ssh root@flap.id 'rm -rf /mnt/volume1/armbian_build/userpatches/overlay/*.tar.gz'
        - find . -mindepth 2 -name docker-compose.yml | xargs -I{} yq -r '.services | to_entries[]| .value.image' {} | xargs -I{} ssh root@flap.id 'docker pull --platform=armhf {}; docker save {} | gzip > /mnt/volume1/armbian_build/userpatches/overlay/$(basename {}).tar.gz'

        # Clean previous output folder.
        - ssh root@flap.id 'rm -rf /mnt/volume1/armbian_build/output/images/*'

        # Build the image on the remote server.
        - ssh root@flap.id '/mnt/volume1/armbian_build/compile.sh BOARD=odroidxu4 BRANCH=current RELEASE=buster BUILD_MINIMAL=yes BUILD_DESKTOP=no KERNEL_ONLY=no KERNEL_CONFIGURE=no'

        # Compress the image.
        - ssh root@flap.id 'gzip --fast --keep /mnt/volume1/armbian_build/output/images/Armbian_20.05.0-trunk_Odroidxu4_buster_current_5.4.23_minimal.img'

        # Store the image file in the root account.
        - ssh root@flap.id "mv /mnt/volume1/armbian_build/output/images/Armbian_20.05.0-trunk_Odroidxu4_buster_current_5.4.23_minimal.img.gz /root/flap-$CI_COMMIT_REF_SLUG-$(date +"%Y.%m.%d")-odroidxu4.img.gz"

deploy_doc:
    stage: deploy
    needs: []
    rules:
        - if: '$CI_COMMIT_BRANCH == "master"'
    script:
        - apk add rclone
        - rclone sync --verbose --config "$RCLONE_CONFIG" ./system/docs  scaleway:docs.flap.cloud
        - rclone sync --config "$RCLONE_CONFIG" --exclude '**/node_modules/' --exclude ''.git/ ./  scaleway:docs.flap.cloud/src