Skip to content

Commit

Permalink
reworking EVERYTHING
Browse files Browse the repository at this point in the history
  • Loading branch information
@miloswrath
miloswrath committed Sep 16, 2024
1 parent 00194e5 commit 9378724
Show file tree
Hide file tree
Showing 6 changed files with 336 additions and 59 deletions.
287 changes: 232 additions & 55 deletions .github/workflows/main.yml
View file
Original file line number Diff line number Diff line change
@@ -1,103 +1,280 @@
name: Task1 QC
# .github/workflows/deploy.yml

name: Task1 QC and Deploy

on:
push:
branches:
- main

permissions:
contents: write
packages: write
issues: write
id-token: write
pages: write

jobs:
process_raw:
name: Process Raw CSV Files
runs-on: self-hosted
outputs:
sub: ${{ steps.set_vars.outputs.sub }}
task: ${{ steps.set_vars.outputs.task }}
version: ${{ steps.set_vars.outputs.version }}
sub: ${{ steps.set_vars.outputs.sub }}
task: ${{ steps.set_vars.outputs.task }}
version: ${{ steps.set_vars.outputs.version }}
run_part: ${{ steps.set_vars.outputs.run_part }}

steps:
- name: checkout code and return recently uploaded file in /data
- name: Checkout Code
uses: actions/checkout@v3
- name: Get changed files

- name: Get Changed CSV Files
run: |
#!/bin/bash
#!/bin/bash
# Get the list of CSV files changed in the last 24 hours
data=$(git log --since="24 hours ago" --name-only --pretty=format: -- '*.csv' | sort | uniq)
# Get the list of CSV files changed in the last 24 hours
data=$(git log --since="24 hours ago" --name-only --pretty=format: -- '*.csv' | sort | uniq)
# Export the data variable to the environment
echo "data=$data" >> $GITHUB_ENV
# Export the data variable to the environment
echo "data=$data" >> $GITHUB_ENV
# Print the changed CSV files
echo "Changed CSV files in the last 24 hours: $data"
# Print the changed CSV files
echo "Changed CSV files in the last 24 hours: $data"
- name: set up python
- name: Install Python Dependencies
run: |
python -m pip install --upgrade pip
- name: parse raw
- name: Parse Raw CSV Files
id: set_vars
run: |
# Loop through each CSV file in $data
for file in $data; do
filename=$(basename "$file")
IFS='_' read -r sub task version <<< "$filename"
version="${version%.csv}" # Remove the .csv extension from version
echo "::set-output name=sub::$sub"
echo "::set-output name=task::$task"
echo "::set-output name=version::$version"
echo "Subject: $sub"
echo "Task: $task"
echo "Version: $version"
done
# Loop through each CSV file in $data
for file in $data; do
# Extract the directory and filename
dir=$(dirname "$file")
filename=$(basename "$file")
# Extract the run-* part from the directory
run_part=$(basename "$dir")
# Split the filename into sub, task, and version
IFS='_' read -r sub task version <<< "$filename"
version="${version%.csv}" # Remove the .csv extension from version
# Set outputs
echo "::set-output name=run_part::$run_part"
echo "::set-output name=sub::$sub"
echo "::set-output name=task::$task"
echo "::set-output name=version::$version"
# Print the extracted values
echo "Run Part: $run_part"
echo "Subject: $sub"
echo "Task: $task"
echo "Version: $version"
done
run_qc:
name: Run Quality Control
runs-on: self-hosted
needs: process_raw

steps:
- name: Checkout code
- name: Checkout Code
uses: actions/checkout@v3

- name: Debug env vars
- name: Debug Environment Variables
run: |
echo "sub=${{ needs.process_raw.outputs.sub }}"
echo "task=${{ needs.process_raw.outputs.task }}"
echo "version=${{ needs.process_raw.outputs.version }}"
echo "Subject: ${{ needs.process_raw.outputs.sub }}"
echo "Task: ${{ needs.process_raw.outputs.task }}"
echo "Version: ${{ needs.process_raw.outputs.version }}"
echo "Run Part: ${{ needs.process_raw.outputs.run_part }}"
- name: run quality control
- name: Install Python Dependencies
run: |
sub=${{ needs.process_raw.outputs.sub }}
task=${{ needs.process_raw.outputs.task }}
vers=${{ needs.process_raw.outputs.version }}
for sub in ${sub}; do
python -m pip install --upgrade pip
- name: Run Quality Control Script
run: |
sub=${{ needs.process_raw.outputs.sub }}
task=${{ needs.process_raw.outputs.task }}
vers=${{ needs.process_raw.outputs.version }}
run_part=${{ needs.process_raw.outputs.run_part }}
for sub in ${sub}; do
echo "Processing subject: $sub"
for task in ${task}; do
echo "Processing task: $task"
for vers in ${vers}; do
echo "Processing task: $task"
for vers in ${vers}; do
echo "Processing version: $vers"
csv_file="./data/${sub}/processed/${sub}_${task}_${vers}.csv"
log_file="./data/${sub}/qc_${task}_${vers}.log"
csv_file="./data/${sub}/processed/${run_part}/${sub}_${task}_${vers}.csv"
mkdir -p "./data/${sub}/${run_part}"
log_file="./data/${sub}/${run_part}/qc_${task}_${vers}.log"
echo "CSV file: $csv_file"
echo "Log file: $log_file"
if [ -f "$csv_file" ]; then
python ./code/ATSqC.py -s "$csv_file" -o "./data/${sub}/" -sub "$sub" | tee "$log_file"
echo "QC for ${sub}_${task}_${vers} running"
python ./code/ATSqC.py -s "$csv_file" -o "./data/${sub}/${run_part}" -sub "$sub" | tee "$log_file"
echo "QC for ${sub}_${task}_${vers} completed"
else
echo "CSV file $csv_file does not exist"
echo "CSV file $csv_file does not exist"
fi
done
done
done
done
done
- name: Commit and Push QC Results
run: |
git config --global user.name "miloswrath"
git config --global user.email "[email protected]"
git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY
git add ./data/*/*/*.png
git commit -m "Add QC results for subjects $(date +%Y-%m-%d)"
git push origin main
- name: List Directory After QC
run: |
echo "Listing directory after running QC:"
find ./data -type d
find ./data -type f
add:
name: Generate Jekyll Posts and Deploy
runs-on: ubuntu-latest
needs: [process_raw, run_qc]

push:
runs-on: self-hosted
needs: run_qc
steps:
- name: Commit and Push Changes
# 1. Checkout the Repository

- name: Checkout Repository
uses: actions/checkout@v4
with:
persist-credentials: false # Recommended for security
fetch-depth: 0
ref: main
- name: Pull Latest Changes
run: git pull origin main

# 4. Set Up Ruby Environment
- name: Setup Ruby
uses: ruby/setup-ruby@v1
with:
ruby-version: '3.1' # Specify your Ruby version
bundler-cache: true # Caches installed gems automatically
cache-version: 1 # Increment if you need to reset the cache

# 5. Install Ruby Dependencies
- name: Install Dependencies
run: bundle install

# 6. Generate Jekyll Posts from PNGs
- name: Generate Jekyll Posts
run: |
set -e # Exit immediately if a command exits with a non-zero status
set -x # Print commands and their arguments as they are executed
run_part=${{ needs.process_raw.outputs.run_part }}
POSTS_DIR="_posts"
mkdir -p "$POSTS_DIR" # Ensure the _posts directory exists

# Initialize an associative array to group images by subject
declare -A subjects

# Get the list of PNG files added in the latest commit
images=$(git diff --name-only HEAD~1 HEAD | grep '\.png$')
echo "Images: $images"

# Iterate over each PNG file and group them by subject number
for file in $images; do
# Check if the file exists to avoid errors
if [ ! -f "$file" ]; then
continue
fi

# Extract the subject number from the filename (assuming it's the first part before '_')
filename=$(basename "$file")
subject=$(echo "$filename" | awk -F_ '{print $1}')

# Append the filename to the subject's array
subjects["$subject"]+="$file "
done

# Generate Jekyll posts for each subject
for subject in "${!subjects[@]}"; do
# Define the post filename with current date and subject number
timestamp=$(date +%H%M%S)
post_filename="$POSTS_DIR/$(date +%Y-%m-%d)-subject-$subject-$timestamp.md"

# Create the Jekyll post
{
echo "---"
echo "layout: post"
echo "title: Subject $subject"
echo "date: $(date +%Y-%m-%d)"
echo "categories: subjects"
echo "---"
echo ""
# Add images to the post
for image in ${subjects["$subject"]}; do
# Adjust the image path based on your site structure
# Assuming images are served from the root
echo "![]($image)"
done
} > "$post_filename"

echo "Created post: $post_filename"
done
# 7. List _posts Directory for Verification (Optional)
- name: List _posts Directory
run: |
echo "Listing _posts directory:"
ls -la _posts
# 8. Commit and Push Generated Posts
- name: Commit and Push Posts
run: |
git config --global user.name "miloswrath"
git config --global user.email "[email protected]"
git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY
git add .
git commit -m "Automated commit by GitHub Actions"
git push
# Add new posts to git
git add _posts/*.md

# Commit changes if there are any
if ! git diff --cached --exit-code > /dev/null; then
git commit -m "Add new posts for subjects $(date +%Y-%m-%d)"
git push origin main # Replace 'main' with your default branch if different
else
echo "No changes to commit."
fi
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

# 9. Build the Jekyll Site
- name: Build with Jekyll
run: bundle exec jekyll build --verbose --baseurl "${{ github.event.inputs.base_path || '' }}"
env:
GITHUB_TOKEN: ${{ secrets.GIT_PAT }}
JEKYLL_ENV: production

# 10. Deploy to GitHub Pages
# Using GitHub's built-in Pages action
- name: Configure GitHub Pages
uses: actions/configure-pages@v5

- name: Upload Pages Artifact
uses: actions/upload-pages-artifact@v1
with:
path: ./_site # Ensure this matches your Jekyll build output

- name: List _site Directory
run: |
echo "Listing _site directory:"
ls -la _site
- name: Deploy to GitHub Pages
uses: actions/deploy-pages@v1
with:
token: ${{ secrets.GITHUB_TOKEN }}
5 changes: 5 additions & 0 deletions Gemfile
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# Gemfile teehee with no extension teehee

source 'https://rubygems.org'

gem 'github-pages', group: :jekyll_plugins
14 changes: 14 additions & 0 deletions _config.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
title: "Boost Behavioral Data -> ATS"
description: "A blog showcasing various subjects post-QC."
baseurl: "/boost-beh-ATS" # If your site is at https://username.github.io, leave it empty. If at https://username.github.io/repo-name, set it to "/repo-name"
url: "https://hbclab.github.io" # Replace with your GitHub Pages URL
logo: https://avatars0.githubusercontent.com/u/24659915?s=400&u=12a4f626488fe0f692d77f355d9dd9f3e4e63f7a&v=4
repository: "HBClab/boost-beh-ATS" # Replace with your GitHub repository in "owner/re
plugins:
- jekyll-feed
- jekyll-seo-tag
- jekyll-github-metadata
include:
- data

theme: minima
13 changes: 13 additions & 0 deletions _layouts/default.html
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>{{ page.title }} | {{ site.title }}</title>
<meta name="viewport" content="width=device-width, initial-scale=1">
<!-- Link to external CSS -->
<link rel="stylesheet" href="{{ '/assets/css/style.css' | relative_url }}">
</head>
<body>
{{ content }}
</body>
</html>
Loading

0 comments on commit 9378724

Please sign in to comment.