How return artifact from Child Job to Parent pipeline?

Question

Use trigger for dynamic select test job

prepare_test:
  image: $CI_REGISTRY/platform/docker-images/vault:1.8
  variables:
    CONTEXT_TEST: |
      include:
      # PRODUCT
        - project: 'gitlabci/integration-test'
          ref: dev_v2
          file: 
            - 'spark/.base_integration_test.yml'
            - 'spark/.base_integration_test_with_kafka.yml'   
      integration_test:
        variables:
          COVERAGE_SOURCE: "./src"  
    INTEGRATION_TEST: |
      $CONTEXT_TEST
        extends: .base_integration_test
    INTEGRATION_TEST_WITH_KAFKA: |
      $CONTEXT_TEST 
        extends: .base_integration_test_with_kafka  
  stage: prepare_test
  script:
    - export CICD_KAFKA_HOST=$(cat test/fixtures.py | grep KAFKA_HOST)
    - >
      if [ "$CICD_KAFKA_HOST" != "" ]; then
        export CICD_KAFKA_HOST="true"
        echo "$INTEGRATION_TEST_WITH_KAFKA" >> test.yml
      else
        export CICD_KAFKA_HOST="false"
        echo "$INTEGRATION_TEST" >> test.yml
      fi
    - env | sort -f
  artifacts:
    paths:
      - test.yml
    expire_in: 6000 seconds


# --------------- Integration test --------------- ###
integration_test:
  stage: test 
  trigger:
    include:
      - artifact: test.yml
        job: prepare_test
    strategy: depend

after complete child integration_test create coverage-report.xml

How return coverage-report.xml to parent pipeline?

Simplistic

Answer 1

You can use Gitlab API:

my-job:
  image: ...
  stage: ...
  script:
    - >
        export CI_CHILD_PIPELINE_ID=$(curl --header "PRIVATE-TOKEN: $GITLAB_USER_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/pipelines/$CI_PIPELINE_ID/bridges" | jq ".[].downstream_pipeline.id")
    - echo $CI_CHILD_PIPELINE_ID
    - >
        export CI_CHILD_JOB_ID=$(curl --header "PRIVATE-TOKEN: $GITLAB_USER_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/pipelines/$CI_CHILD_PIPELINE_ID/jobs" | jq '.[].id')
    - echo $CI_CHILD_JOB_ID 
    - 'curl --output artifacts.zip --header "PRIVATE-TOKEN: $GITLAB_USER_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$CI_CHILD_JOB_ID/artifacts"'
    - unzip artifacts.zip

Simplicity

Answer 2

In case someone need it, this would download and create folders for every child pipeline under a folder:

# Expose artifacts from the child pipeline
npm-expose-artifacts:
  stage: test
  image: ...
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: always
    - when: never  # If no conditions above are met, don't run
  script:
    - echo "Running npm expose artifacts job"
    - apt-get update && apt-get install -y curl jq unzip fastjar gawk file
    - curl --version
    - jq --version
    - echo "Publishing artifacts"
    - echo "GitLab User Token [REDACTED]"
    - echo "$CI_API_V4_URL"
    - echo "$GITLAB_USER_TOKEN"

    # Fetch the child pipeline ID with added logging
    - echo "Fetching child pipeline ID..."
    - echo "$CI_API_V4_URL/projects/$CI_PROJECT_ID/pipelines/$CI_PIPELINE_ID/bridges"
    - >
      export BRIDGES_RESPONSE=$(curl --silent --header "PRIVATE-TOKEN: $GL_TOKEN"
      "$CI_API_V4_URL/projects/$CI_PROJECT_ID/pipelines/$CI_PIPELINE_ID/bridges")
    - echo "Bridges Response $BRIDGES_RESPONSE"
    - >
      export CI_CHILD_PIPELINE_ID=$(echo "$BRIDGES_RESPONSE" | jq -r ".[0].downstream_pipeline.id" 
      2>/dev/null || echo "null")
    - >
      if [ "$CI_CHILD_PIPELINE_ID" = "null" ] || [ -z "$CI_CHILD_PIPELINE_ID" ]; then
        echo "Error: Child pipeline ID not found or failed to parse response.";
        exit 1;
      fi
    - echo "Child Pipeline ID  $CI_CHILD_PIPELINE_ID"

    # Fetch the job ID from the child pipeline with added logging
    - echo "Fetching child job ID... $CI_API_V4_URL/projects/$CI_PROJECT_ID/pipelines/$CI_CHILD_PIPELINE_ID/jobs"
    - >
      export JOBS_RESPONSE=$(curl --silent --header "PRIVATE-TOKEN: $GL_TOKEN" 
      "$CI_API_V4_URL/projects/$CI_PROJECT_ID/pipelines/$CI_CHILD_PIPELINE_ID/jobs")
    - echo "Jobs Response  $JOBS_RESPONSE"
    - >
      export CI_CHILD_JOB_ID=$(echo "$JOBS_RESPONSE" | jq -r '.[0].id' 
      2>/dev/null || echo "null")
    - >
      if [ "$CI_CHILD_JOB_ID" = "null" ] || [ -z "$CI_CHILD_JOB_ID" ]; then
        echo "Error: Child job ID not found or failed to parse response.";
        exit 1;
      fi
    - echo "Child Job ID  $CI_CHILD_JOB_ID"


    - >
      export URLS_ARTIFACTS=$(echo "$JOBS_RESPONSE" | jq -r --arg ci_api_v4_url "$CI_API_V4_URL" '
          .[] |
          .project.id as $project_id |
          .name as $name |
          .id as $job_id |
          "\($ci_api_v4_url)/projects/27363/jobs/\($job_id)/artifacts/ \($name)"
      ')
    
    - echo "$URLS_ARTIFACTS"

    - DOWNLOAD_DIR=".artifacts-download"
    - mkdir -p "$DOWNLOAD_DIR"

    - echo "List of URLs to download:"
    - echo "$URLS_ARTIFACTS"
    - cd "$DOWNLOAD_DIR"
    - >
      echo "$URLS_ARTIFACTS" | while IFS= read -r line; do
        ls -la "."
        # Split the line into URL and filename
        # url=$(echo "$line" | awk '{print $1}')
        # filename=$(echo "$line" | awk '{print $2}')
        echo "start loop"
        echo "line is $line"
        line=$(echo "$line" | xargs)
        echo "line is $line"
        read -r url job_name <<< "$line"

        echo "url is $url"
        echo "job_name is $job_name"

          STATUS_CODE=$(curl -H "JOB-TOKEN: $CI_JOB_TOKEN" -o /dev/null -s -w "%{http_code}" "$url")
          if [ "$STATUS_CODE" -eq 404 ]; then
            echo "404 ignoring $job_name"
          else
            mkdir "$job_name"
            cd "$job_name"
            # Download the artifact using curl
            echo "Downloading $filename from $url..."
            # curl -v -L -H "PRIVATE-TOKEN: $GL_TOKEN" "$url&job_token=$CI_JOB_TOKEN"
            echo "$url"
            #curl -L -H "PRIVATE-TOKEN: $GL_TOKEN" "$url" -o "artifacts.zip"
            # curl -L -H "PRIVATE-TOKEN: $CI_JOB_TOKEN" "$url" -o "artifacts.zip"
            curl -L -H "JOB-TOKEN: $CI_JOB_TOKEN" "$url" -o "artifacts.zip"

            ls -la .
            echo "Extracting ZIP  artifacts.zip"
            file artifacts.zip
            jar tf artifacts.zip
            jar xvf artifacts.zip
            ls -la .

            cd ..
          fi

        echo "-----------------------"
      done
    - cd ..
    - ls -la "$DOWNLOAD_DIR"
    - find "$DOWNLOAD_DIR" | sed -e "s/[^-][^\/]*\//  |/g" -e "s/|\([^ ]\)/|-\1/"

    - echo "Listing files..."
    - ls -la
    - find ./.artifacts-download | sed -e "s/[^-][^\/]*\//  |/g" -e "s/|\([^ ]\)/|-\1/"
  needs:
    - npm-create-jobs
  artifacts:
    when: on_success
    paths:
      - .artifacts-download
    reports:
      junit:
        - .artifacts-download/changethis.test.xml 

Gonick