diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fef13ef..a442700 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,13 +1,326 @@ -# You can override the included template(s) by including variable overrides -# SAST customization: https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings -# Secret Detection customization: https://docs.gitlab.com/ee/user/application_security/secret_detection/#customizing-settings -# Dependency Scanning customization: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#customizing-the-dependency-scanning-settings -# Container Scanning customization: https://docs.gitlab.com/ee/user/application_security/container_scanning/#customizing-the-container-scanning-settings -# Note that environment variables can be set in several places -# See https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence stages: -- test -sast: - stage: test + # - build + - test + - sast + # - dast + # - tag + # - deploy + include: -- template: Security/SAST.gitlab-ci.yml + - template: Jobs/Container-Scanning.gitlab-ci.yml + # - template: Jobs/Dependency-Scanning.gitlab-ci.yml + - local: .gitlab/Dependency-Scanning.v2.gitlab-ci.yml + - template: Jobs/SAST-IaC.gitlab-ci.yml + - template: Jobs/SAST.gitlab-ci.yml + - template: Jobs/Secret-Detection.gitlab-ci.yml + +default: + retry: 1 + +# variables: +# BASE_IMAGE_NAME: $CI_REGISTRY_IMAGE +# TRIVY_CACHE_DIR: .cache/trivy +# TRIVY_NO_PROGRESS: "true" +# TRIVY_TIMEOUT: "10m0s" +# TRIVY_USERNAME: $CI_REGISTRY_USER +# TRIVY_PASSWORD: $CI_REGISTRY_PASSWORD +# TRIVY_REGISTRY: $CI_REGISTRY +# UV_PROJECT_ENVIRONMENT: .venv +# UV_CACHE_DIR: .cache/uv +# BUILDAH_ISOLATION: oci +# STORAGE_DRIVER: vfs +# DOCKER_HOST: "tcp://docker:2375" +# DOCKER_TLS_CERTDIR: "" + +# cache: +# key: "${CI_COMMIT_REF_SLUG}" +# paths: +# - $TRIVY_CACHE_DIR +# - $UV_CACHE_DIR +# - $UV_PROJECT_ENVIRONMENT +# policy: pull-push + +.docker-job: &docker-job + image: docker:cli + services: + - docker:dind + before_script: + - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + +.buildah-job: &buildah-job + image: quay.io/containers/buildah:latest + variables: + STORAGE_DRIVER: vfs + before_script: + - buildah login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + +.trivy-fs-template: &trivy-fs-scan + image: + name: aquasec/trivy:latest + entrypoint: [""] + cache: + paths: + - $TRIVY_CACHE_DIR + policy: pull-push + before_script: + - mkdir -p $TRIVY_CACHE_DIR + script: + - trivy filesystem --skip-files $TRIVY_CACHE_DIR --format cyclonedx --output fs-sbom.json . + - trivy filesystem --skip-files $TRIVY_CACHE_DIR --format sarif --output gl-sast-fs-report.json . + allow_failure: true + artifacts: + reports: + sast: gl-sast-fs-report.json + paths: + - fs-sbom.json + - gl-sast-fs-report.json + expire_in: 1 week + when: always + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_COMMIT_TAG + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + - if: $SAST_DISABLED + when: never + +# .trivy-image-template: &trivy-image-scan +# stage: security +# image: +# name: aquasec/trivy:latest +# entrypoint: [""] +# cache: +# paths: +# - $TRIVY_CACHE_DIR +# policy: pull-push +# before_script: +# - mkdir -p $TRIVY_CACHE_DIR +# script: +# - | +# trivy image \ +# --scanners vuln \ +# --format cyclonedx \ +# --output image-sbom-${IMAGE_TYPE}.json \ +# $IMAGE_NAME:$CI_COMMIT_SHA +# - | +# trivy image \ +# --format sarif \ +# --output gl-sast-image-${IMAGE_TYPE}-report.json \ +# $IMAGE_NAME:$CI_COMMIT_SHA +# allow_failure: true +# artifacts: +# reports: +# sast: gl-sast-image-${IMAGE_TYPE}-report.json +# paths: +# - image-sbom-${IMAGE_TYPE}.json +# - gl-sast-image-${IMAGE_TYPE}-report.json +# expire_in: 1 week +# when: always +# rules: +# - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH +# - if: $CI_COMMIT_TAG +# - if: $CI_PIPELINE_SOURCE == 'merge_request_event' +# - if: $SAST_DISABLED +# when: never + +# .webhook-template: &webhook-config +# image: curlimages/curl:latest +# script: +# - | +# curl -sf -X POST \ +# -H "Content-Type: application/json" \ +# -H "Authorization: Bearer $WEBHOOK_SECRET_TOKEN" \ +# -H "Webhook-Identifier: $WEBHOOK_BYPASS_TOKEN" \ +# "$WEBHOOK_URL" +# rules: +# - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH +# when: on_success + +.build-template: &build-config + <<: *buildah-job + stage: build + script: + - | + buildah bud \ + --tag $IMAGE_NAME:$CI_COMMIT_SHA \ + --file $CONTAINERFILE \ + --target $BUILDTARGET \ + --layers \ + --cache-from $IMAGE_NAME-cache \ + --cache-to $IMAGE_NAME-cache \ + . + - buildah push $IMAGE_NAME:$CI_COMMIT_SHA + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + when: always + - if: $CI_COMMIT_TAG + when: always + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + when: manual + allow_failure: true + +.tag-template: &tag-config + <<: *buildah-job + stage: tag + script: + - | + set -euo pipefail + + buildah pull $IMAGE_NAME:$CI_COMMIT_SHA + + if [ -n "${CI_COMMIT_TAG:-}" ]; then + buildah tag $IMAGE_NAME:$CI_COMMIT_SHA $IMAGE_NAME:$CI_COMMIT_TAG + buildah push $IMAGE_NAME:$CI_COMMIT_TAG + fi + + if [ -n "${CI_COMMIT_BRANCH:-}" ]; then + buildah tag $IMAGE_NAME:$CI_COMMIT_SHA $IMAGE_NAME:$CI_COMMIT_REF_SLUG + buildah push $IMAGE_NAME:$CI_COMMIT_REF_SLUG + + if [ "$CI_COMMIT_BRANCH" = "$CI_DEFAULT_BRANCH" ]; then + buildah tag $IMAGE_NAME:$CI_COMMIT_SHA $IMAGE_NAME:latest + buildah push $IMAGE_NAME:latest + fi + fi + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_COMMIT_TAG + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + when: manual + allow_failure: true + +# build-runtime: +# <<: *build-config +# variables: +# IMAGE_NAME: $BASE_IMAGE_NAME/backend +# CONTAINERFILE: Containerfile +# BUILDTARGET: runtime + +# build-tests: +# <<: *build-config +# variables: +# IMAGE_NAME: $BASE_IMAGE_NAME/backend-tests +# CONTAINERFILE: Containerfile +# BUILDTARGET: tests + +# build-migrations: +# <<: *build-config +# variables: +# IMAGE_NAME: $BASE_IMAGE_NAME/backend-migrations +# CONTAINERFILE: Containerfile +# BUILDTARGET: migrations + +# build-ml: +# <<: *build-config +# when: manual +# variables: +# IMAGE_NAME: $BASE_IMAGE_NAME/ml +# CONTAINERFILE: Containerfile +# BUILDTARGET: ml + +# lint: +# <<: *uv-job +# stage: test +# script: +# - source $HOME/.local/bin/env +# - uv sync --group linters --frozen +# - source $UV_PROJECT_ENVIRONMENT/bin/activate +# - just lint +# allow_failure: true +# rules: +# - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH +# - if: $CI_PIPELINE_SOURCE == 'merge_request_event' +# - if: $CI_COMMIT_TAG + +# test: +# <<: *docker-job +# stage: test +# tags: +# - beta +# variables: +# COMPOSE_PROFILES: | +# --profile migrations +# --profile tests +# --profile ml +# script: +# - apk add --no-cache docker-compose +# - export PROFILES="$(printf '%s ' $COMPOSE_PROFILES)" +# - cp "$TEST_STAGE_FIREBASE_CONF" ./infrastructure/configs/backend/firebase.json +# - | +# ( +# while true; do +# docker compose -f compose.yaml $PROFILES logs -f 2>&1 +# sleep 1 +# done +# ) | tee -a compose.log & +# - LOGS_PID=$! +# - | +# REGISTRY_PREFIX=$CI_REGISTRY_IMAGE IMAGE_TAG=$CI_COMMIT_SHA \ +# docker compose -f compose.yaml -f compose.prod.yaml \ +# $PROFILES up -d --quiet-pull --quiet-build 2>&1 | tee compose.log +# - | +# TEST_CONTAINER_ID=$(docker compose -f compose.yaml $PROFILES ps -q tests -a) +# timeout 600 docker wait $TEST_CONTAINER_ID +# TEST_EXIT_CODE=$(docker inspect --format "{{.State.ExitCode}}" $TEST_CONTAINER_ID) + +# if [ $TEST_EXIT_CODE -eq 0 ]; then +# echo "Tests passed." +# else +# echo "Tests failed with exit code $TEST_EXIT_CODE." +# exit 1 +# fi +# - | +# docker compose -f compose.yaml $PROFILES down +# - cat .cov/coverage.txt +# artifacts: +# paths: +# - ./.cov +# - ./compose.log +# reports: +# coverage_report: +# coverage_format: cobertura +# path: .cov/coverage.xml +# expire_in: 1 week +# when: always +# coverage: /TOTAL.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/ +# rules: +# - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH +# - if: $CI_PIPELINE_SOURCE == 'merge_request_event' +# dependencies: +# - build-runtime +# - build-tests +# - build-migrations + +sast-filesystem: + <<: *trivy-fs-scan + +# sast-image-: +# <<: *trivy-image-scan +# variables: +# IMAGE_NAME: $BASE_IMAGE_NAME/backend +# IMAGE_TYPE: runtime +# dependencies: +# - build-runtime + +# tag-backend: +# <<: *tag-config +# variables: +# IMAGE_NAME: $BASE_IMAGE_NAME/backend + +# webhook-backend-deploy: +# <<: *webhook-config +# stage: deploy +# variables: +# WEBHOOK_URL: $WEBHOOK_URL_BACKEND +# environment: +# name: staging +# url: https://datarush.itqdev.xyz +# resource_group: staging +# dependencies: +# - build-runtime +# - sast-image-runtime + +workflow: + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_TAG diff --git a/.gitlab/Dependency-Scanning.v2.gitlab-ci.yml b/.gitlab/Dependency-Scanning.v2.gitlab-ci.yml new file mode 100644 index 0000000..1cbfbfb --- /dev/null +++ b/.gitlab/Dependency-Scanning.v2.gitlab-ci.yml @@ -0,0 +1,120 @@ +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/ +# +# Configure dependency scanning with spec:inputs (https://docs.gitlab.com/ci/yaml/#specinputs). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#spec_inputs +spec: + inputs: + # Variables for job behavior. + job_name: + type: string + default: 'dependency-scanning' + description: "The name of the dependency scanning job." + stage: + type: string + default: test + description: "The stage of the dependency scanning job." + allow_failure: + type: boolean + default: true + description: "Whether the dependency scanning job failure should fail the pipeline." + # Default value set to SECURE_ANALYZERS_PREFIX to ensure that global variables applying to AST templates + # work as expected. If $SECURE_ANALYZERS_PREFIX is set globally and this input is supplied, the global + # variable will be preferred. + analyzer_image_prefix: + type: string + default: "$SECURE_ANALYZERS_PREFIX" + description: "The registry url prefix pointing to the repository of the analyzer." + analyzer_image_name: + type: string + default: "dependency-scanning" + description: "The repository of the analyzer image used by the dependency-scanning job." + analyzer_image_version: + type: string + default: "1" + description: "The version of the analyzer image used by the dependency-scanning job." + + # Variables for analyzer behavior. + additional_ca_cert_bundle: + type: string + default: "" + description: "CA certificate bundle to trust. The CA bundle provided here is added to the system's certificates and also used by other tools during the scanning process." + pipcompile_requirements_file_name_pattern: + type: string + default: "" + description: "Custom requirements file name pattern to use when analyzing. The pattern should match file names only, not directory paths. See https://github.com/bmatcuk/doublestar/tree/v1#patterns for syntax details." + max_scan_depth: + type: number + default: 2 + description: "Defines how many directory levels analyzer should search for supported files. A value of -1 means the analyzer will search all directories regardless of depth." + excluded_paths: + type: string + default: "**/spec,**/test,**/tests,**/tmp,**/node_modules,**/.bundle,**/vendor,**/.git" + description: "A comma-separated list of paths (globs supported) to exclude from the scan." + include_dev_dependencies: + type: boolean + default: true + description: "Include development/test dependencies when scanning a supported file." + enable_static_reachability: + type: boolean + default: false + description: "Enable static reachability." + analyzer_log_level: + type: string + default: "info" + description: "Logging level used by the analyzer. The options are fatal, error, warn, info, debug." + enable_vulnerability_scan: + type: boolean + default: true + description: "Enable the vulnerability analysis of generated SBOMs." + vulnerability_scan_api_timeout: + type: number + default: 10 + description: "Vulnerability scanning API request timeout in seconds." + vulnerability_scan_api_download_delay: + type: number + default: 3 + description: "Vulnerability scanning API initial delay in seconds before downloading scan results." + +--- + +variables: + # Setting this variable affects all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" + # Variable used to track the template/compopnent introducing DS analyzer. Please do not override. + # This is an internal variable + INTERNAL_DS_ANALYZER_CI_INTEGRATION: "template::v2" + +"$[[ inputs.job_name ]]": + variables: + DS_ANALYZER_IMAGE: $[[ inputs.analyzer_image_prefix ]]/$[[ inputs.analyzer_image_name ]]:$[[ inputs.analyzer_image_version ]] + image: "$DS_ANALYZER_IMAGE" + stage: $[[ inputs.stage ]] + allow_failure: $[[ inputs.allow_failure ]] + script: + - echo "Using Dependency-Scanning.v2.gitlab-ci.yml" + # Variable fallback for legacy variables used by version 1 of the Dependency Scanning template. + - export ADDITIONAL_CA_CERT_BUNDLE="${ADDITIONAL_CA_CERT_BUNDLE:-$[[ inputs.additional_ca_cert_bundle ]]}" + - export DS_PIPCOMPILE_REQUIREMENTS_FILE_NAME_PATTERN="${DS_PIPCOMPILE_REQUIREMENTS_FILE_NAME_PATTERN:-$[[ inputs.pipcompile_requirements_file_name_pattern ]]}" + - export DS_MAX_DEPTH="${DS_MAX_DEPTH:-$[[ inputs.max_scan_depth ]]}" + - export DS_EXCLUDED_PATHS="${DS_EXCLUDED_PATHS:-$[[ inputs.excluded_paths ]]}" + - export DS_INCLUDE_DEV_DEPENDENCIES="${DS_INCLUDE_DEV_DEPENDENCIES:-$[[ inputs.include_dev_dependencies ]]}" + - export DS_STATIC_REACHABILITY_ENABLED="${DS_STATIC_REACHABILITY_ENABLED:-$[[ inputs.enable_static_reachability ]]}" + - export SECURE_LOG_LEVEL="${SECURE_LOG_LEVEL:-$[[ inputs.analyzer_log_level ]]}" + - export DS_ENABLE_VULNERABILITY_SCAN="${DS_ENABLE_VULNERABILITY_SCAN:-$[[ inputs.enable_vulnerability_scan ]]}" + - export DS_API_TIMEOUT="${DS_API_TIMEOUT:-$[[ inputs.vulnerability_scan_api_timeout ]]}" + - export DS_API_SCAN_DOWNLOAD_DELAY="${DS_API_SCAN_DOWNLOAD_DELAY:-$[[ inputs.vulnerability_scan_api_download_delay ]]}" + - /analyzer run + artifacts: + access: "developer" + paths: + - "**/gl-sbom-*.cdx.json" + - "gl-dependency-scanning-report.json" + reports: + cyclonedx: "**/gl-sbom-*.cdx.json" + dependency_scanning: 'gl-dependency-scanning-report.json' + rules: + - if: '($AST_ENABLE_MR_PIPELINES == "true" || $AST_ENABLE_MR_PIPELINES == null) && $CI_PIPELINE_SOURCE == "merge_request_event"' + - if: '($AST_ENABLE_MR_PIPELINES == "true" || $AST_ENABLE_MR_PIPELINES == null) && $CI_OPEN_MERGE_REQUESTS' + when: never + - if: $CI_COMMIT_BRANCH