2024-09-17 17:12:22.213604 new snippets

seeker/report.txt

@@ -1,3 +1,23 @@
+--------------------------------------------------------------------------------
+ 2024-09-17 17:12:22.213604
+--------------------------------------------------------------------------------
+  On branch main
+Your branch is up to date with 'origin/main'.
+
+Untracked files:
+  (use "git add <file>..." to include in what will be committed)
+	snippet/clone-github.sh
+	snippet/data-exploration.py
+	snippet/data-preprocessing.py
+	snippet/dataset-preparation.py
+	snippet/find-lambda.sh
+	snippet/fix-atime.py
+	snippet/install.sh
+	snippet/main.py
+	snippet/read-dataset.py
+
+nothing added to commit but untracked files present (use "git add" to track)
+
 --------------------------------------------------------------------------------
  2024-09-16 17:12:34.795202
 --------------------------------------------------------------------------------

seeker/snippet/clone-github.sh

@@ -0,0 +1,49 @@
+#date: 2024-09-17T16:55:11Z
+#url: https://api.github.com/gists/ecec01f35abce4af34a2409af65dc85a
+#owner: https://api.github.com/users/dleslie
+
+#!/bin/bash
+
+username="${username:=dleslie}"
+
+function git_clone_or_update {
+    local url=$1
+    local name="${url##*/}";
+    local name="${name%.git}";
+    if [ -d $name ]; then
+        pushd $name
+        git pull
+        popd
+    else
+        git clone --recursive $url $name
+    fi
+}
+
+found_urls=""
+function get_type {
+    local type=$1
+    local page=$2
+    found_urls=`curl -s https://api.github.com/users/$username/$type?per_page=100\&page=$page | jq -c '.[] | .ssh_url'`
+}
+
+function fetch_type {
+    local type=$1
+    local root=$2
+    local index=1
+
+    mkdir -p $root
+    pushd $root
+    get_type $type $index
+    while [ ! -z "$found_urls" ]; do
+        for url in $found_urls; do
+            url=`echo $url | tr -d \"`;
+            git_clone_or_update $url;
+        done
+        index=$((index+1));
+        get_type $type $index
+    done
+    popd
+}
+
+fetch_type repos $username
+fetch_type starred starred

seeker/snippet/data-exploration.py

@@ -0,0 +1,13 @@
+#date: 2024-09-17T17:09:06Z
+#url: https://api.github.com/gists/262f97965307559af70c6cdf976ea6a0
+#owner: https://api.github.com/users/docsallover
+
+# How many articles per subject?
+print(data.groupby(['subject'])['text'].count())
+data.groupby(['subject'])['text'].count().plot(kind="bar")
+plt.show()
+
+# How many fake and real articles?
+print(data.groupby(['target'])['text'].count())
+data.groupby(['target'])['text'].count().plot(kind="bar")
+plt.show()

seeker/snippet/data-preprocessing.py

@@ -0,0 +1,35 @@
+#date: 2024-09-17T16:59:16Z
+#url: https://api.github.com/gists/af16acf0a46155283562cb074498248c
+#owner: https://api.github.com/users/docsallover
+
+from nltk.corpus import stopwords
+nltk.download('stopwords')  # Download stopwords if not already downloaded
+
+# Remove the date column (assuming it's not relevant for analysis)
+data.drop(["date"], axis=1, inplace=True)
+print(data.head())
+
+# Remove the title column (focusing on text content)
+data.drop(["title"], axis=1, inplace=True)
+print(data.head())
+
+# Convert text to lowercase for consistency
+data['text'] = data['text'].apply(lambda x: x.lower())
+print(data.head())
+
+import string
+
+# Remove punctuation for cleaner analysis
+def punctuation_removal(text):
+    all_list = [char for char in text if char not in string.punctuation]
+    clean_str = ''.join(all_list)
+    return clean_str
+
+data['text'] = data['text'].apply(punctuation_removal)
+print(data.head())
+
+# Remove stopwords for better word representation
+stop = stopwords.words('english')
+
+data['text'] = data['text'].apply(lambda x: ' '.join([word for word in x.split() if word not in (stop)]))
+print(data.head())

seeker/snippet/dataset-preparation.py

@@ -0,0 +1,20 @@
+#date: 2024-09-17T16:52:09Z
+#url: https://api.github.com/gists/68d5b273d0473a793c5e192956037efd
+#owner: https://api.github.com/users/docsallover
+
+# Add a flag to track fake and real news
+fake['target'] = 'fake'
+true['target'] = 'true'
+
+# Concatenate the dataframes
+data = pd.concat([fake, true]).reset_index(drop=True)
+
+# Check the shape of the combined dataset
+print(data.shape)
+
+# Shuffle the data
+data = shuffle(data)
+data = data.reset_index(drop=True)
+
+# Check the first few rows of the shuffled data
+print(data.head())

seeker/snippet/find-lambda.sh

@@ -0,0 +1,161 @@
+#date: 2024-09-17T17:00:19Z
+#url: https://api.github.com/gists/3d35243c3b968b9d1d883680ec0ea2fe
+#owner: https://api.github.com/users/darksinge
+
+#!/usr/bin/env bash
+
+set -e
+
+NO_CACHE=0
+CLEAN=0
+
+AWS_PROFILE=""
+AWS_REGION="us-east-1"
+
+help() {
+    cat << EOF
+Usage: $(basename "$0") [OPTIONS]
+
+This script helps you find and open an AWS Lambda function in the AWS Console.
+
+Options:
+  -p, --profile PROFILE   Specify the AWS profile to use
+  -r, --region  REGION    Specify the AWS region to use
+  --no-cache              Disable caching of AWS resources
+  --clean                 Clear out cached results
+  -h, --help              Display this help message and exit
+
+The script will prompt you to:
+1. Choose or enter a stage (dev/stage/prod/other)
+2. Select a CloudFormation stack
+3. Choose a Lambda function from the selected stack
+
+It will then open the chosen Lambda function in your default web browser.
+
+Note: This script requires the following tools to be installed:
+- AWS CLI
+- jq
+- gum (for interactive prompts)
+EOF
+}
+
+if ! gum -v >/dev/null 2>&1; then
+  echo "The 'gum' command was not found."
+  echo "Visit https://github.com/charmbracelet/gum for installation instructions."
+  exit 1
+fi
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -p|--profile)
+      export AWS_PROFILE="$2"
+      shift 2
+      ;;
+    --no-cache)
+      export NO_CACHE=1
+      shift
+      ;;
+    --region)
+      AWS_REGION="$2"
+      shift 2
+      ;;
+    --clean)
+      CLEAN=1
+      ;;
+    -h|--help)
+      help
+      exit 0
+      ;;
+    *)
+      help
+      exit 1
+      ;;
+  esac
+done
+
+CACHE_DIR="$HOME/.cache/find-fn"
+if [ ! -d "$CACHE_DIR" ]; then
+  mkdir -p "$CACHE_DIR"
+fi
+
+if [ $CLEAN -eq 1 ]; then
+  rm -rf "$CACHE_DIR" >/dev/null 2>&1
+  mkdir -p "$CACHE_DIR"
+  exit 0
+fi
+
+STAGE=$(gum choose "dev" "stage" "prod" "other")
+if [ "$STAGE" == "other" ]; then
+  STAGE=$(gum input --placeholder "stage name?")
+fi
+
+STACKS_LIST_CACHE="$CACHE_DIR/$STAGE-stacks"
+
+function _make_temp() {
+  type="$1"
+  fcache="$CACHE_DIR/$STAGE-$type"
+
+  if [ $NO_CACHE -eq 1 ]; then
+    echo "$(mktemp)"
+    return 0
+  fi
+
+  local tmp=""
+  if [ -f "$fcache" ]; then
+    tmp=$(cat "$fcache")
+  fi
+
+  if [ ! -f "$tmp" ]; then
+    tmp=$(mktemp)
+    echo "$tmp" > "$fcache"
+  else
+    tmp=$(cat "$fcache")
+  fi
+
+  echo "$tmp"
+}
+
+function make_temp() {
+  set +e
+  echo $(_make_temp "$1")
+  set -e
+}
+
+stack_list_cache=$(make_temp "stacks")
+if [ -f "$stack_list_cache" ]; then
+  STACKS=$(cat "$stack_list_cache")
+fi
+
+if [ -z "$STACKS" ]; then
+  STACKS=$(gum spin --spinner dot --title 'Fetching stacks' --show-output -- \
+            aws cloudformation list-stacks \
+            --query "StackSummaries[?starts_with(StackName, '$STAGE-certifications-service-')].StackName" \
+            --output json)
+
+  echo "$STACKS" > "$stack_list_cache"
+fi
+
+PREFIX="$STAGE-certifications-service-"
+STACK_NAME=$(gum choose $(echo "$STACKS" | jq -r '.[]' | sed "s/$PREFIX//"))
+STACK_NAME="$PREFIX$STACK_NAME"
+
+resource_cache=$(make_temp "$STACK_NAME-resources")
+if [ -f "$resource_cache" ]; then
+  RESOURCES=$(cat "$resource_cache")
+fi
+
+if [ -z "$RESOURCES" ]; then
+  RESOURCES=$(gum spin --spinner dot --title 'Fetching resources' --show-output -- \
+              aws cloudformation list-stack-resources --stack-name "$STACK_NAME" \
+              --output json)
+  echo "$RESOURCES" > "$resource_cache"
+fi
+
+RESOURCES=$(cat "$resource_cache" | jq '.StackResourceSummaries')
+
+LOGICAL_ID=$(echo "$RESOURCES" | jq -r '.[] | select(.ResourceType == "AWS::Lambda::Function") | .LogicalResourceId' | gum filter)
+PHYSICAL_ID=$(echo "$RESOURCES" | jq -r ".[] | select(.LogicalResourceId == \"$LOGICAL_ID\") | .PhysicalResourceId")
+
+if [ -n "$PHYSICAL_ID" ]; then
+  open "https://$AWS_REGION.console.aws.amazon.com/lambda/home?region=$AWS_REGION#/functions/$PHYSICAL_ID?tab=monitor"
+fi

seeker/snippet/fix-atime.py

@@ -0,0 +1,24 @@
+#date: 2024-09-17T17:09:25Z
+#url: https://api.github.com/gists/10ab9650a38e8556c74a1dd4876cd60c
+#owner: https://api.github.com/users/meeb
+
+#!/usr/bin/python3
+
+import os
+import time
+from pathlib import Path
+
+this_dir = os.path.dirname(os.path.realpath(__file__))
+silly_time = time.mktime(time.strptime('2038-01-01', '%Y-%m-%d'))
+
+for root, dirs, files in os.walk(this_dir):
+    rootpath = Path(root)
+    for f in files:
+        filepath = rootpath / f
+        if not os.path.isfile(filepath):
+            continue
+        access_time = os.path.getatime(filepath)
+        # if the file access time is > 2038-01-01 then touch it
+        if access_time > silly_time:
+            print(f'Fixing future access time file: {filepath} ({access_time} > {silly_time})')
+            filepath.touch()

seeker/snippet/install.sh

@@ -0,0 +1,25 @@
+#date: 2024-09-17T16:55:05Z
+#url: https://api.github.com/gists/18888e23fee5a7788305f1aa35a1df3b
+#owner: https://api.github.com/users/zitterbewegung
+
+#!/bin/bash
+
+# Make the script executable
+chmod +x install.sh
+
+# Install Python dependencies
+echo "Installing Python dependencies from requirements.txt..."
+pip install -r requirements.txt
+
+# Check and install Ghidra if needed
+if ! command -v ghidra &> /dev/null; then
+    echo "Ghidra not found. Please install Ghidra manually from https://ghidra-sre.org/"
+fi
+
+# Check and install DTrace if needed
+if ! command -v dtrace &> /dev/null; then
+    echo "DTrace not found. Please install DTrace using your system's package manager."
+fi
+
+# Add any other specific dependency checks as needed
+echo "All dependencies installed. You may need to restart your terminal for some changes to take effect."