Automated Commit -> New Data #22

Workflow file for this run

	# .github/workflows/deploy.yml

	name: Task1 QC and Deploy

	on:
	push:
	branches:
	- main

	permissions:
	contents: write
	packages: write
	issues: write
	id-token: write
	pages: write

	jobs:
	process_raw:
	name: Process Raw CSV Files
	runs-on: self-hosted
	outputs:
	sub: ${{ steps.set_vars.outputs.sub }}
	task: ${{ steps.set_vars.outputs.task }}
	version: ${{ steps.set_vars.outputs.version }}
	run_part: ${{ steps.set_vars.outputs.run_part }}

	steps:
	- name: Checkout Code
	uses: actions/checkout@v3

	- name: Get Changed CSV Files
	run: \|
	#!/bin/bash

	# Get the list of CSV files changed in the last 24 hours
	data=$(git log --since="24 hours ago" --name-only --pretty=format: -- '*.csv' \| sort \| uniq)

	# Convert the list into a multi-line format
	formatted_data=$(echo "$data" \| tr ' ' '\n')

	# Export the formatted data variable to the environment
	echo "data<<EOF" >> $GITHUB_ENV
	echo "$formatted_data" >> $GITHUB_ENV
	echo "EOF" >> $GITHUB_ENV

	# Print the changed CSV files
	echo "Changed CSV files in the last 24 hours:"
	echo "$formatted_data"


	- name: Install Python Dependencies
	run: \|
	python -m pip install --upgrade pip

	- name: Parse Raw CSV Files
	id: set_vars
	run: \|
	# Loop through each CSV file in $data
	for file in $data; do
	# Extract the directory and filename
	dir=$(dirname "$file")
	filename=$(basename "$file")

	# Extract the run-* part from the directory
	run_part=$(basename "$dir")

	# Split the filename into sub, task, and version
	IFS='_' read -r sub task version <<< "$filename"
	version="${version%.csv}" # Remove the .csv extension from version

	# Set outputs
	echo "::set-output name=run_part::$run_part"
	echo "::set-output name=sub::$sub"
	echo "::set-output name=task::$task"
	echo "::set-output name=version::$version"

	# Print the extracted values
	echo "Run Part: $run_part"
	echo "Subject: $sub"
	echo "Task: $task"
	echo "Version: $version"
	done

	run_qc:
	name: Run Quality Control
	runs-on: self-hosted
	needs: process_raw

	steps:
	- name: Checkout Code
	uses: actions/checkout@v3

	- name: Debug Environment Variables
	run: \|
	echo "Subject: ${{ needs.process_raw.outputs.sub }}"
	echo "Task: ${{ needs.process_raw.outputs.task }}"
	echo "Version: ${{ needs.process_raw.outputs.version }}"
	echo "Run Part: ${{ needs.process_raw.outputs.run_part }}"

	- name: Install Python Dependencies
	run: \|
	python -m pip install --upgrade pip

	- name: Run Quality Control Script
	env:
	TEASE: ${{ secrets.TEASE }}
	run: \|
	sub=${{ needs.process_raw.outputs.sub }}
	task=${{ needs.process_raw.outputs.task }}
	vers=${{ needs.process_raw.outputs.version }}
	run_part=${{ needs.process_raw.outputs.run_part }}

	for sub in ${sub}; do
	echo "Processing subject: $sub"
	for task in ${task}; do
	echo "Processing task: $task"
	for vers in ${vers}; do
	echo "Processing version: $vers"
	csv_file="./data/${sub}/processed/${run_part}/${sub}_${task}_${vers}.csv"
	mkdir -p "./data/${sub}/${run_part}"
	log_file="./data/${sub}/${run_part}/qc_${task}_${vers}.log"
	echo "CSV file: $csv_file"
	echo "Log file: $log_file"
	if [ -f "$csv_file" ]; then
	python ./code/NFqC.py -s "$csv_file" -o "./data/${sub}/${run_part}" -sub "$sub" \| tee "$log_file"
	echo "QC for ${sub}_${task}_${vers} completed"
	else
	echo "CSV file $csv_file does not exist"
	fi
	done
	done
	done


	- name: Commit and Push QC Results
	run: \|
	git config --global user.name "miloswrath"
	git config --global user.email "[email protected]"
	git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY
	git add ./data///*.png
	git commit -m "Add QC results for subjects $(date +%Y-%m-%d)"
	git push origin main

	- name: List Directory After QC
	run: \|
	echo "Listing directory after running QC:"
	find ./data -type d
	find ./data -type f



	add:
	name: Generate Jekyll Posts and Deploy
	runs-on: ubuntu-latest
	needs: [process_raw, run_qc]

	steps:
	# 1. Checkout the Repository

	- name: Checkout Repository
	uses: actions/checkout@v4
	with:
	persist-credentials: false # Recommended for security
	fetch-depth: 0
	ref: main
	- name: Pull Latest Changes
	run: git pull origin main

	# 4. Set Up Ruby Environment
	- name: Setup Ruby
	uses: ruby/setup-ruby@v1
	with:
	ruby-version: '3.1' # Specify your Ruby version
	bundler-cache: true # Caches installed gems automatically
	cache-version: 1 # Increment if you need to reset the cache

	# 5. Install Ruby Dependencies
	- name: Install Dependencies
	run: bundle install

	# 6. Generate Jekyll Posts from PNGs
	- name: Generate Jekyll Posts
	run: \|
	set -e # Exit immediately if a command exits with a non-zero status
	set -x # Print commands and their arguments as they are executed

	run_part=${{ needs.process_raw.outputs.run_part }}
	POSTS_DIR="_posts"
	mkdir -p "$POSTS_DIR" # Ensure the _posts directory exists

	# Initialize an associative array to group images by subject
	declare -A subjects

	# Get the list of PNG files added in the latest commit
	images=$(git diff --name-only HEAD~1 HEAD \| grep '\.png$')
	echo "Images: $images"

	# Iterate over each PNG file and group them by subject number
	for file in $images; do
	# Check if the file exists to avoid errors
	if [ ! -f "$file" ]; then
	continue
	fi

	# Extract the subject number from the filename (assuming it's the first part before '_')
	filename=$(basename "$file")
	subject=$(echo "$filename" \| awk -F_ '{print $1}')

	# Append the filename to the subject's array
	subjects["$subject"]+="$file "
	done

	# Generate Jekyll posts for each subject
	for subject in "${!subjects[@]}"; do
	# Define the post filename with current date and subject number
	timestamp=$(date +%H%M%S)
	post_filename="$POSTS_DIR/$(date +%Y-%m-%d)-subject-$subject-$timestamp.md"

	# Create the Jekyll post
	{
	echo "---"
	echo "layout: post"
	echo "title: Subject $subject"
	echo "date: $(date +%Y-%m-%d)"
	echo "categories: subjects"
	echo "---"
	echo ""
	# Add images to the post
	for image in ${subjects["$subject"]}; do
	# Adjust the image path based on your site structure
	# Assuming images are served from the root
	echo "![]($image)"
	done
	} > "$post_filename"

	echo "Created post: $post_filename"
	done
	# 7. List _posts Directory for Verification (Optional)
	- name: List _posts Directory
	run: \|
	echo "Listing _posts directory:"
	ls -la _posts

	# 8. Commit and Push Generated Posts
	- name: Commit and Push Posts
	run: \|
	git config --global user.name "miloswrath"
	git config --global user.email "[email protected]"
	git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY

	# Add new posts to git
	git add _posts/*.md

	# Commit changes if there are any
	if ! git diff --cached --exit-code > /dev/null; then
	git commit -m "Add new posts for subjects $(date +%Y-%m-%d)"
	git push origin main # Replace 'main' with your default branch if different
	else
	echo "No changes to commit."
	fi
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

	# 9. Build the Jekyll Site
	- name: Build with Jekyll
	run: bundle exec jekyll build --verbose --baseurl "${{ github.event.inputs.base_path \|\| '' }}"
	env:
	JEKYLL_ENV: production

	# 10. Deploy to GitHub Pages
	# Using GitHub's built-in Pages action
	- name: Configure GitHub Pages
	uses: actions/configure-pages@v5

	- name: Upload Pages Artifact
	uses: actions/upload-pages-artifact@v1
	with:
	path: ./_site # Ensure this matches your Jekyll build output

	- name: List _site Directory
	run: \|
	echo "Listing _site directory:"
	ls -la _site

	- name: Deploy to GitHub Pages
	uses: actions/deploy-pages@v1
	with:
	token: ${{ secrets.GITHUB_TOKEN }}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Automated Commit -> New Data #22

Workflow file

Automated Commit -> New Data #22

Jobs

Run details

Workflow file for this run