ci: added gitlab templates

2025-12-14 13:48:45 +03:00
parent e1cc77dc12
commit 9f08279a77
2 changed files with 444 additions and 11 deletions
@@ -1,13 +1,326 @@
-# You can override the included template(s) by including variable overrides
-# SAST customization: https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings
-# Secret Detection customization: https://docs.gitlab.com/ee/user/application_security/secret_detection/#customizing-settings
-# Dependency Scanning customization: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#customizing-the-dependency-scanning-settings
-# Container Scanning customization: https://docs.gitlab.com/ee/user/application_security/container_scanning/#customizing-the-container-scanning-settings
-# Note that environment variables can be set in several places
-# See https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence
 stages:
- test
-sast:
-  stage: test
+  # - build
+  - test
+  - sast
+  # - dast
+  # - tag
+  # - deploy
+
 include:
- template: Security/SAST.gitlab-ci.yml
+  - template: Jobs/Container-Scanning.gitlab-ci.yml
+  # - template: Jobs/Dependency-Scanning.gitlab-ci.yml
+  - local: .gitlab/Dependency-Scanning.v2.gitlab-ci.yml
+  - template: Jobs/SAST-IaC.gitlab-ci.yml
+  - template: Jobs/SAST.gitlab-ci.yml
+  - template: Jobs/Secret-Detection.gitlab-ci.yml
+
+default:
+  retry: 1
+
+# variables:
+#   BASE_IMAGE_NAME: $CI_REGISTRY_IMAGE
+#   TRIVY_CACHE_DIR: .cache/trivy
+#   TRIVY_NO_PROGRESS: "true"
+#   TRIVY_TIMEOUT: "10m0s"
+#   TRIVY_USERNAME: $CI_REGISTRY_USER
+#   TRIVY_PASSWORD: $CI_REGISTRY_PASSWORD
+#   TRIVY_REGISTRY: $CI_REGISTRY
+#   UV_PROJECT_ENVIRONMENT: .venv
+#   UV_CACHE_DIR: .cache/uv
+#   BUILDAH_ISOLATION: oci
+#   STORAGE_DRIVER: vfs
+#   DOCKER_HOST: "tcp://docker:2375"
+#   DOCKER_TLS_CERTDIR: ""
+
+# cache:
+#   key: "${CI_COMMIT_REF_SLUG}"
+#   paths:
+#     - $TRIVY_CACHE_DIR
+#     - $UV_CACHE_DIR
+#     - $UV_PROJECT_ENVIRONMENT
+#   policy: pull-push
+
+.docker-job: &docker-job
+  image: docker:cli
+  services:
+    - docker:dind
+  before_script:
+    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
+
+.buildah-job: &buildah-job
+  image: quay.io/containers/buildah:latest
+  variables:
+    STORAGE_DRIVER: vfs
+  before_script:
+    - buildah login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
+
+.trivy-fs-template: &trivy-fs-scan
+  image:
+    name: aquasec/trivy:latest
+    entrypoint: [""]
+  cache:
+    paths:
+      - $TRIVY_CACHE_DIR
+    policy: pull-push
+  before_script:
+    - mkdir -p $TRIVY_CACHE_DIR
+  script:
+    - trivy filesystem --skip-files $TRIVY_CACHE_DIR --format cyclonedx --output fs-sbom.json .
+    - trivy filesystem --skip-files $TRIVY_CACHE_DIR --format sarif --output gl-sast-fs-report.json .
+  allow_failure: true
+  artifacts:
+    reports:
+      sast: gl-sast-fs-report.json
+    paths:
+      - fs-sbom.json
+      - gl-sast-fs-report.json
+    expire_in: 1 week
+    when: always
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $CI_COMMIT_TAG
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+    - if: $SAST_DISABLED
+      when: never
+
+# .trivy-image-template: &trivy-image-scan
+#   stage: security
+#   image:
+#     name: aquasec/trivy:latest
+#     entrypoint: [""]
+#   cache:
+#     paths:
+#       - $TRIVY_CACHE_DIR
+#     policy: pull-push
+#   before_script:
+#     - mkdir -p $TRIVY_CACHE_DIR
+#   script:
+#     - |
+#       trivy image \
+#         --scanners vuln \
+#         --format cyclonedx \
+#         --output image-sbom-${IMAGE_TYPE}.json \
+#         $IMAGE_NAME:$CI_COMMIT_SHA
+#     - |
+#       trivy image \
+#         --format sarif \
+#         --output gl-sast-image-${IMAGE_TYPE}-report.json \
+#         $IMAGE_NAME:$CI_COMMIT_SHA
+#   allow_failure: true
+#   artifacts:
+#     reports:
+#       sast: gl-sast-image-${IMAGE_TYPE}-report.json
+#     paths:
+#       - image-sbom-${IMAGE_TYPE}.json
+#       - gl-sast-image-${IMAGE_TYPE}-report.json
+#     expire_in: 1 week
+#     when: always
+#   rules:
+#     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+#     - if: $CI_COMMIT_TAG
+#     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+#     - if: $SAST_DISABLED
+#       when: never
+
+# .webhook-template: &webhook-config
+#   image: curlimages/curl:latest
+#   script:
+#     - |
+#       curl -sf -X POST \
+#         -H "Content-Type: application/json" \
+#         -H "Authorization: Bearer $WEBHOOK_SECRET_TOKEN" \
+#         -H "Webhook-Identifier: $WEBHOOK_BYPASS_TOKEN" \
+#         "$WEBHOOK_URL"
+#   rules:
+#     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+#       when: on_success
+
+.build-template: &build-config
+  <<: *buildah-job
+  stage: build
+  script:
+    - |
+      buildah bud \
+        --tag $IMAGE_NAME:$CI_COMMIT_SHA \
+        --file $CONTAINERFILE \
+        --target $BUILDTARGET \
+        --layers \
+        --cache-from $IMAGE_NAME-cache \
+        --cache-to $IMAGE_NAME-cache \
+        .
+    - buildah push $IMAGE_NAME:$CI_COMMIT_SHA
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+      when: always
+    - if: $CI_COMMIT_TAG
+      when: always
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+      when: manual
+      allow_failure: true
+
+.tag-template: &tag-config
+  <<: *buildah-job
+  stage: tag
+  script:
+    - |
+      set -euo pipefail
+
+      buildah pull $IMAGE_NAME:$CI_COMMIT_SHA
+
+      if [ -n "${CI_COMMIT_TAG:-}" ]; then
+        buildah tag $IMAGE_NAME:$CI_COMMIT_SHA $IMAGE_NAME:$CI_COMMIT_TAG
+        buildah push $IMAGE_NAME:$CI_COMMIT_TAG
+      fi
+
+      if [ -n "${CI_COMMIT_BRANCH:-}" ]; then
+        buildah tag $IMAGE_NAME:$CI_COMMIT_SHA $IMAGE_NAME:$CI_COMMIT_REF_SLUG
+        buildah push $IMAGE_NAME:$CI_COMMIT_REF_SLUG
+
+        if [ "$CI_COMMIT_BRANCH" = "$CI_DEFAULT_BRANCH" ]; then
+          buildah tag $IMAGE_NAME:$CI_COMMIT_SHA $IMAGE_NAME:latest
+          buildah push $IMAGE_NAME:latest
+        fi
+      fi
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $CI_COMMIT_TAG
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+      when: manual
+      allow_failure: true
+
+# build-runtime:
+#   <<: *build-config
+#   variables:
+#     IMAGE_NAME: $BASE_IMAGE_NAME/backend
+#     CONTAINERFILE: Containerfile
+#     BUILDTARGET: runtime
+
+# build-tests:
+#   <<: *build-config
+#   variables:
+#     IMAGE_NAME: $BASE_IMAGE_NAME/backend-tests
+#     CONTAINERFILE: Containerfile
+#     BUILDTARGET: tests
+
+# build-migrations:
+#   <<: *build-config
+#   variables:
+#     IMAGE_NAME: $BASE_IMAGE_NAME/backend-migrations
+#     CONTAINERFILE: Containerfile
+#     BUILDTARGET: migrations
+
+# build-ml:
+#   <<: *build-config
+#   when: manual
+#   variables:
+#     IMAGE_NAME: $BASE_IMAGE_NAME/ml
+#     CONTAINERFILE: Containerfile
+#     BUILDTARGET: ml
+
+# lint:
+#   <<: *uv-job
+#   stage: test
+#   script:
+#     - source $HOME/.local/bin/env
+#     - uv sync --group linters --frozen
+#     - source $UV_PROJECT_ENVIRONMENT/bin/activate
+#     - just lint
+#   allow_failure: true
+#   rules:
+#     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+#     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+#     - if: $CI_COMMIT_TAG
+
+# test:
+#   <<: *docker-job
+#   stage: test
+#   tags:
+#     - beta
+#   variables:
+#     COMPOSE_PROFILES: |
+#       --profile migrations
+#       --profile tests
+#       --profile ml
+#   script:
+#     - apk add --no-cache docker-compose
+#     - export PROFILES="$(printf '%s ' $COMPOSE_PROFILES)"
+#     - cp "$TEST_STAGE_FIREBASE_CONF" ./infrastructure/configs/backend/firebase.json
+#     - |
+#       (
+#         while true; do
+#           docker compose -f compose.yaml $PROFILES logs -f 2>&1
+#           sleep 1
+#         done
+#       ) | tee -a compose.log &
+#     - LOGS_PID=$!
+#     - |
+#       REGISTRY_PREFIX=$CI_REGISTRY_IMAGE IMAGE_TAG=$CI_COMMIT_SHA \
+#       docker compose -f compose.yaml -f compose.prod.yaml \
+#       $PROFILES up -d --quiet-pull --quiet-build 2>&1 | tee compose.log
+#     - |
+#       TEST_CONTAINER_ID=$(docker compose -f compose.yaml $PROFILES ps -q tests -a)
+#       timeout 600 docker wait $TEST_CONTAINER_ID
+#       TEST_EXIT_CODE=$(docker inspect --format "{{.State.ExitCode}}" $TEST_CONTAINER_ID)
+
+#       if [ $TEST_EXIT_CODE -eq 0 ]; then
+#         echo "Tests passed."
+#       else
+#         echo "Tests failed with exit code $TEST_EXIT_CODE."
+#         exit 1
+#       fi
+#     - |
+#       docker compose -f compose.yaml $PROFILES down
+#     - cat .cov/coverage.txt
+#   artifacts:
+#     paths:
+#       - ./.cov
+#       - ./compose.log
+#     reports:
+#       coverage_report:
+#         coverage_format: cobertura
+#         path: .cov/coverage.xml
+#     expire_in: 1 week
+#     when: always
+#   coverage: /TOTAL.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/
+#   rules:
+#     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+#     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+#   dependencies:
+#     - build-runtime
+#     - build-tests
+#     - build-migrations
+
+sast-filesystem:
+  <<: *trivy-fs-scan
+
+# sast-image-:
+#   <<: *trivy-image-scan
+#   variables:
+#     IMAGE_NAME: $BASE_IMAGE_NAME/backend
+#     IMAGE_TYPE: runtime
+#   dependencies:
+#     - build-runtime
+
+# tag-backend:
+#   <<: *tag-config
+#   variables:
+#     IMAGE_NAME: $BASE_IMAGE_NAME/backend
+
+# webhook-backend-deploy:
+#   <<: *webhook-config
+#   stage: deploy
+#   variables:
+#     WEBHOOK_URL: $WEBHOOK_URL_BACKEND
+#   environment:
+#     name: staging
+#     url: https://datarush.itqdev.xyz
+#   resource_group: staging
+#   dependencies:
+#     - build-runtime
+#     - sast-image-runtime
+
+workflow:
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    - if: $CI_COMMIT_TAG
@@ -0,0 +1,120 @@
+# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/
+#
+# Configure dependency scanning with spec:inputs (https://docs.gitlab.com/ci/yaml/#specinputs).
+# List of available variables: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#spec_inputs
+spec:
+  inputs:
+    # Variables for job behavior.
+    job_name:
+      type: string
+      default: 'dependency-scanning'
+      description: "The name of the dependency scanning job."
+    stage:
+      type: string
+      default: test
+      description: "The stage of the dependency scanning job."
+    allow_failure:
+      type: boolean
+      default: true
+      description: "Whether the dependency scanning job failure should fail the pipeline."
+    # Default value set to SECURE_ANALYZERS_PREFIX to ensure that global variables applying to AST templates
+    # work as expected. If $SECURE_ANALYZERS_PREFIX is set globally and this input is supplied, the global
+    # variable will be preferred.
+    analyzer_image_prefix:
+      type: string
+      default: "$SECURE_ANALYZERS_PREFIX"
+      description: "The registry url prefix pointing to the repository of the analyzer."
+    analyzer_image_name:
+      type: string
+      default: "dependency-scanning"
+      description: "The repository of the analyzer image used by the dependency-scanning job."
+    analyzer_image_version:
+      type: string
+      default: "1"
+      description: "The version of the analyzer image used by the dependency-scanning job."
+
+    # Variables for analyzer behavior.
+    additional_ca_cert_bundle:
+      type: string
+      default: ""
+      description: "CA certificate bundle to trust. The CA bundle provided here is added to the system's certificates and also used by other tools during the scanning process."
+    pipcompile_requirements_file_name_pattern:
+      type: string
+      default: ""
+      description: "Custom requirements file name pattern to use when analyzing. The pattern should match file names only, not directory paths. See https://github.com/bmatcuk/doublestar/tree/v1#patterns for syntax details."
+    max_scan_depth:
+      type: number
+      default: 2
+      description: "Defines how many directory levels analyzer should search for supported files. A value of -1 means the analyzer will search all directories regardless of depth."
+    excluded_paths:
+      type: string
+      default: "**/spec,**/test,**/tests,**/tmp,**/node_modules,**/.bundle,**/vendor,**/.git"
+      description: "A comma-separated list of paths (globs supported) to exclude from the scan."
+    include_dev_dependencies:
+      type: boolean
+      default: true
+      description: "Include development/test dependencies when scanning a supported file."
+    enable_static_reachability:
+      type: boolean
+      default: false
+      description: "Enable static reachability."
+    analyzer_log_level:
+      type: string
+      default: "info"
+      description: "Logging level used by the analyzer. The options are fatal, error, warn, info, debug."
+    enable_vulnerability_scan:
+      type: boolean
+      default: true
+      description: "Enable the vulnerability analysis of generated SBOMs."
+    vulnerability_scan_api_timeout:
+      type: number
+      default: 10
+      description: "Vulnerability scanning API request timeout in seconds."
+    vulnerability_scan_api_download_delay:
+      type: number
+      default: 3
+      description: "Vulnerability scanning API initial delay in seconds before downloading scan results."
+
+---
+
+variables:
+  # Setting this variable affects all Security templates
+  # (SAST, Dependency Scanning, ...)
+  SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products"
+  # Variable used to track the template/compopnent introducing DS analyzer. Please do not override.
+  # This is an internal variable
+  INTERNAL_DS_ANALYZER_CI_INTEGRATION: "template::v2"
+
+"$[[ inputs.job_name ]]":
+  variables:
+    DS_ANALYZER_IMAGE: $[[ inputs.analyzer_image_prefix ]]/$[[ inputs.analyzer_image_name ]]:$[[ inputs.analyzer_image_version ]]
+  image: "$DS_ANALYZER_IMAGE"
+  stage: $[[ inputs.stage ]]
+  allow_failure: $[[ inputs.allow_failure ]]
+  script:
+    - echo "Using Dependency-Scanning.v2.gitlab-ci.yml"
+    # Variable fallback for legacy variables used by version 1 of the Dependency Scanning template.
+    - export ADDITIONAL_CA_CERT_BUNDLE="${ADDITIONAL_CA_CERT_BUNDLE:-$[[ inputs.additional_ca_cert_bundle ]]}"
+    - export DS_PIPCOMPILE_REQUIREMENTS_FILE_NAME_PATTERN="${DS_PIPCOMPILE_REQUIREMENTS_FILE_NAME_PATTERN:-$[[ inputs.pipcompile_requirements_file_name_pattern ]]}"
+    - export DS_MAX_DEPTH="${DS_MAX_DEPTH:-$[[ inputs.max_scan_depth ]]}"
+    - export DS_EXCLUDED_PATHS="${DS_EXCLUDED_PATHS:-$[[ inputs.excluded_paths ]]}"
+    - export DS_INCLUDE_DEV_DEPENDENCIES="${DS_INCLUDE_DEV_DEPENDENCIES:-$[[ inputs.include_dev_dependencies ]]}"
+    - export DS_STATIC_REACHABILITY_ENABLED="${DS_STATIC_REACHABILITY_ENABLED:-$[[ inputs.enable_static_reachability ]]}"
+    - export SECURE_LOG_LEVEL="${SECURE_LOG_LEVEL:-$[[ inputs.analyzer_log_level ]]}"
+    - export DS_ENABLE_VULNERABILITY_SCAN="${DS_ENABLE_VULNERABILITY_SCAN:-$[[ inputs.enable_vulnerability_scan ]]}"
+    - export DS_API_TIMEOUT="${DS_API_TIMEOUT:-$[[ inputs.vulnerability_scan_api_timeout ]]}"
+    - export DS_API_SCAN_DOWNLOAD_DELAY="${DS_API_SCAN_DOWNLOAD_DELAY:-$[[ inputs.vulnerability_scan_api_download_delay ]]}"
+    - /analyzer run
+  artifacts:
+    access: "developer"
+    paths:
+      - "**/gl-sbom-*.cdx.json"
+      - "gl-dependency-scanning-report.json"
+    reports:
+      cyclonedx: "**/gl-sbom-*.cdx.json"
+      dependency_scanning: 'gl-dependency-scanning-report.json'
+  rules:
+    - if: '($AST_ENABLE_MR_PIPELINES == "true" || $AST_ENABLE_MR_PIPELINES == null) && $CI_PIPELINE_SOURCE == "merge_request_event"'
+    - if: '($AST_ENABLE_MR_PIPELINES == "true" || $AST_ENABLE_MR_PIPELINES == null) && $CI_OPEN_MERGE_REQUESTS'
+      when: never
+    - if: $CI_COMMIT_BRANCH