diff --git a/seeker/report.txt b/seeker/report.txt index 7f12445e..1491f572 100644 --- a/seeker/report.txt +++ b/seeker/report.txt @@ -1,3 +1,28 @@ +-------------------------------------------------------------------------------- + 2024-07-09 17:12:50.821088 +-------------------------------------------------------------------------------- + On branch main +Your branch is up to date with 'origin/main'. + +Untracked files: + (use "git add ..." to include in what will be committed) + snippet/LAB0001.java + snippet/RedisTemplate.java + snippet/aws-nuke.sh + snippet/change_user.sh + snippet/create-cache.java + snippet/filename.java + snippet/get-cache.java + snippet/get-latest-terraform.sh + snippet/install_csi_tool.sh + snippet/migrate.sh + snippet/qt5-build-for-qgis-macos.sh + snippet/solution.go + snippet/transformer.py + snippet/uuid7.py + +nothing added to commit but untracked files present (use "git add" to track) + -------------------------------------------------------------------------------- 2024-07-08 17:11:24.479467 -------------------------------------------------------------------------------- diff --git a/seeker/snippet/LAB0001.java b/seeker/snippet/LAB0001.java new file mode 100644 index 00000000..49161990 --- /dev/null +++ b/seeker/snippet/LAB0001.java @@ -0,0 +1,13 @@ +//date: 2024-07-09T17:08:26Z +//url: https://api.github.com/gists/bdfc2d74930bca6cc495831620eeabdd +//owner: https://api.github.com/users/Kat-Rambo + +package EX_070724; + +public class LAB0001 { + public static void main(String[] args) { + System.out.println("Hello World!"); + + + } +} diff --git a/seeker/snippet/RedisTemplate.java b/seeker/snippet/RedisTemplate.java new file mode 100644 index 00000000..13592d0d --- /dev/null +++ b/seeker/snippet/RedisTemplate.java @@ -0,0 +1,15 @@ +//date: 2024-07-09T17:09:38Z +//url: https://api.github.com/gists/7b28c84aaa859f0d7118fbf2bf070ab8 +//owner: https://api.github.com/users/enrinal + +public RedisTemplate redisTemplateByte( + LettuceConnectionFactory defaultLettuceConnectionFactory) { + RedisTemplate template = new RedisTemplate<>(); + template.setKeySerializer(RedisSerializer.string()); + template.setHashKeySerializer(RedisSerializer.string()); + template.setValueSerializer(RedisSerializer.byteArray()); + template.setHashValueSerializer(RedisSerializer.byteArray()); + template.setConnectionFactory(defaultLettuceConnectionFactory); + template.afterPropertiesSet(); + return template; +} diff --git a/seeker/snippet/aws-nuke.sh b/seeker/snippet/aws-nuke.sh new file mode 100644 index 00000000..cd02d5eb --- /dev/null +++ b/seeker/snippet/aws-nuke.sh @@ -0,0 +1,33 @@ +#date: 2024-07-09T16:59:18Z +#url: https://api.github.com/gists/5b08d61c03d2396eaa8ec0650d36415f +#owner: https://api.github.com/users/bruteforks + +#!/usr/bin/env bash + +# Set the target directory +TARGET_DIR="$HOME/.local/bin" + +# Fetch the latest release version +latest_version=$(curl -s https://api.github.com/repos/rebuy-de/aws-nuke/releases/latest | grep -oP '"tag_name": "\K(.*)(?=")') + +# Remove the 'v' prefix from the version number +version=${latest_version#v} + +# Construct the download URL +download_url="https://github.com/rebuy-de/aws-nuke/releases/download/${latest_version}/aws-nuke-${latest_version}-linux-amd64.tar.gz" + +# Download the tar.gz file +curl -L -o "aws-nuke-${version}.tar.gz" "$download_url" +echo "Downloaded aws-nuke-${version}.tar.gz" + +echo "Extracting it to $TARGET_DIR" +tar -xz -C "$TARGET_DIR" -f "aws-nuke-${version}.tar.gz" + +mv "$TARGET_DIR/aws-nuke-${latest_version}-linux-amd64" "$TARGET_DIR/aws-nuke" +echo "Renaming binary" + +echo "Removing tar file" +rm -f "aws-nuke-${version}.tar.gz" + +echo "aws-nuke has been installed in $TARGET_DIR" +echo "awsnuke-config in ~/.dotfiles" diff --git a/seeker/snippet/change_user.sh b/seeker/snippet/change_user.sh new file mode 100644 index 00000000..cd655711 --- /dev/null +++ b/seeker/snippet/change_user.sh @@ -0,0 +1,24 @@ +#date: 2024-07-09T17:10:55Z +#url: https://api.github.com/gists/81cdae4248a6a66d62716f40e03ba7f9 +#owner: https://api.github.com/users/mabsboza + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 [personal|work1|work2]" + exit 1 +fi + +if [ "$1" == "personal" ]; then + git config --global user.name "Tu Nombre" + git config --global user.email "tu.email@dominio.com" +elif [ "$1" == "chile" ]; then + git config --global user.name "Tu Nombre Trabajo 1" + git config --global user.email "tu.email.trabajo1@dominio.com" +elif [ "$1" == "usa" ]; then + git config --global user.name "Tu Nombre Trabajo 2" + git config --global user.email "tu.email.trabajo2@dominio.com" +else + echo "Invalid option. Use one of [personal|work1|work2]" + exit 1 +fi + +echo "Git user configuration updated to $1" \ No newline at end of file diff --git a/seeker/snippet/create-cache.java b/seeker/snippet/create-cache.java new file mode 100644 index 00000000..3e8cb78c --- /dev/null +++ b/seeker/snippet/create-cache.java @@ -0,0 +1,19 @@ +//date: 2024-07-09T17:08:59Z +//url: https://api.github.com/gists/12ab04d2ca4e2de5ba5433274f9885e6 +//owner: https://api.github.com/users/enrinal + +@Override +public Mono createCache(HeaderCommonRequest headerCommonRequest, T value, String key, + Duration timeout) { + long startTime = System.nanoTime(); + return Mono.defer(() -> + Mono.fromCallable(() -> Snappy.compress(JSONHelper.convertObjectToJsonInString(value))) + .flatMap(compressedData -> { + redisTemplateByte.opsForValue().set(key, compressedData, timeout); + return Mono.just(true); + }) + .then(Mono.just(true)) + ) + .elapsed() + .hasElement(); +} diff --git a/seeker/snippet/filename.java b/seeker/snippet/filename.java new file mode 100644 index 00000000..c446cc6e --- /dev/null +++ b/seeker/snippet/filename.java @@ -0,0 +1,7 @@ +//date: 2024-07-09T17:12:44Z +//url: https://api.github.com/gists/0f8f49db67e43818accc20177436de16 +//owner: https://api.github.com/users/fav579 + +class sample ( + +) \ No newline at end of file diff --git a/seeker/snippet/get-cache.java b/seeker/snippet/get-cache.java new file mode 100644 index 00000000..c94f4c35 --- /dev/null +++ b/seeker/snippet/get-cache.java @@ -0,0 +1,40 @@ +//date: 2024-07-09T17:03:26Z +//url: https://api.github.com/gists/81141dba7ab189d65ab1acd1d4f8a749 +//owner: https://api.github.com/users/enrinal + +public Mono getCache(HeaderCommonRequest headerCommonRequest, String key, Class clazz) { + return Mono.defer(() -> { + long startTime = System.nanoTime(); + return Mono.defer(() -> + Optional.ofNullable(redisTemplateByte.opsForValue().get(key)) + .map(Mono::just) + .orElseGet(Mono::empty)) + .elapsed() + .flatMap(tupleData -> + Mono.fromCallable(() -> Snappy.uncompressString(tupleData.getT2())) + .flatMap(uncompressedData -> + Mono.defer(() -> { + log.debug("[{}] process time get cache key = {}, process time={} ms", + headerCommonRequest.getMandatoryRequest().getRequestId(), key, + tupleData.getT1()); + return Mono.just( + Objects.requireNonNull( + JSONHelper.convertJsonInStringToObject(uncompressedData, clazz))); + }) + ) + .onErrorResume(throwable -> + Mono.defer(() -> { + cacheMonitoringUtil.logMetric(isSendMetric, "getCache", startTime, FAILED); + log.error("[{}] getCache() - failed get this key={}. with this error={}", + headerCommonRequest.getMandatoryRequest().getRequestId(), key, + throwable); + return Mono.error( + new BusinessLogicException( + CommonResponseCode.PARSE_DATA_ERROR.getCode(), + CommonResponseCode.PARSE_DATA_ERROR.getMessage())); + })) + ) + .doOnSuccess(res -> + cacheMonitoringUtil.logMetric(isSendMetric, "getCache", startTime, SUCCEED)); + }); +} diff --git a/seeker/snippet/get-latest-terraform.sh b/seeker/snippet/get-latest-terraform.sh new file mode 100644 index 00000000..1ef34701 --- /dev/null +++ b/seeker/snippet/get-latest-terraform.sh @@ -0,0 +1,18 @@ +#date: 2024-07-09T17:00:07Z +#url: https://api.github.com/gists/b71c0ec0139df9375ea3f4156979ad2c +#owner: https://api.github.com/users/Malond11 + +#!/bin/bash + +LATEST_RELEASE=$(curl -s https://checkpoint-api.hashicorp.com/v1/check/terraform | jq -r .current_version) +CURRENT_RELEASE=$(terraform version -json | jq -r .terraform_version) +if [[ ${LATEST_RELEASE} != $CURRENT_RELEASE ]]; then + echo "Installing Terraform ${LATEST_RELEASE}..." + + FILENAME=terraform_${LATEST_RELEASE}_linux_amd64.zip + wget https://releases.hashicorp.com/terraform/${LATEST_RELEASE}/$FILENAME + unzip $FILENAME && rm $FILENAME + sudo mv terraform /usr/local/bin +else + echo "Latest version of Terraform (${LATEST_RELEASE}) already installed." +fi \ No newline at end of file diff --git a/seeker/snippet/install_csi_tool.sh b/seeker/snippet/install_csi_tool.sh new file mode 100644 index 00000000..2acef38e --- /dev/null +++ b/seeker/snippet/install_csi_tool.sh @@ -0,0 +1,29 @@ +#date: 2024-07-09T16:57:03Z +#url: https://api.github.com/gists/482f0ab3291143bc8ded9188bc9c2e73 +#owner: https://api.github.com/users/Stop423 + +#!/bin/bash + +# Install development tools +sudo apt update +sudo apt install gcc make linux-headers-$(uname -r) git-core +# Build and install the modified wireless driver +CSITOOL_KERNEL_TAG=csitool-$(uname -r | cut -d . -f 1-2) +git clone https://github.com/posoo/linux-80211n-csitool.git --branch ${CSITOOL_KERNEL_TAG} --depth 1 +cd linux-80211n-csitool +make -C /lib/modules/$(uname -r)/build M=$(pwd)/drivers/net/wireless/iwlwifi modules +sudo make -C /lib/modules/$(uname -r)/build M=$(pwd)/drivers/net/wireless/iwlwifi INSTALL_MOD_DIR=updates \ + modules_install +sudo depmod +cd .. +# Install the modified firmware +git clone https://github.com/posoo/linux-80211n-csitool-supplementary.git +for file in /lib/firmware/iwlwifi-5000-*.ucode +do + sudo mv $file $file.orig +done +sudo cp linux-80211n-csitool-supplementary/firmware/iwlwifi-5000-2.ucode.sigcomm2010 /lib/firmware/ +sudo ln -s iwlwifi-5000-2.ucode.sigcomm2010 /lib/firmware/iwlwifi-5000-2.ucode +# Build the userspace logging tools +make -C linux-80211n-csitool-supplementary/netlink +echo "All Done!" \ No newline at end of file diff --git a/seeker/snippet/migrate.sh b/seeker/snippet/migrate.sh new file mode 100644 index 00000000..a12113ec --- /dev/null +++ b/seeker/snippet/migrate.sh @@ -0,0 +1,156 @@ +#date: 2024-07-09T16:54:05Z +#url: https://api.github.com/gists/bfe99ee6d9061d46609ba171d2705fa2 +#owner: https://api.github.com/users/ArktikAgency + +#!/bin/bash + +# This script will backup your Coolify instance and move everything to a new server. Docker volumes, Coolify database, and ssh keys + +# 1. Script must run on the source server +# 2. Have all the containers running that you want to migrate + +# Configuration - Modify as needed +sshKeyPath="$HOME/.ssh/your_private_key" # Key to destination server +destinationHost="server.example.com" + +# -- Shouldn't need to modify anything below -- +backupSourceDir="/data/coolify/" +backupFileName="coolify_backup.tar.gz" + +# Check if the source directory exists +if [ ! -d "$backupSourceDir" ]; then + echo "❌ Source directory $backupSourceDir does not exist" + exit 1 +fi +echo "✅ Source directory exists" + +# Check if the SSH key file exists +if [ ! -f "$sshKeyPath" ]; then + echo "❌ SSH key file $sshKeyPath does not exist" + exit 1 +fi +echo "✅ SSH key file exists" + +# Check if we can SSH to the destination server, ignore "The authenticity of host can't be established." errors +if ! ssh -i "$sshKeyPath" -o "StrictHostKeyChecking no" -o "ConnectTimeout=5" root@$destinationHost "exit"; then + echo "❌ SSH connection to $destinationHost failed" + exit 1 +fi +echo "✅ SSH connection successful" + +# Get the names of all running Docker containers +containerNames=$(docker ps --format '{{.Names}}') + +# Initialize an empty string to hold the volume paths +volumePaths="" + +# Loop over the container names +for containerName in $containerNames; do + # Get the volumes for the current container + volumeNames=$(docker inspect --format '{{range .Mounts}}{{.Name}}{{end}}' "$containerName") + + # Loop over the volume names + for volumeName in $volumeNames; do + # Check if the volume name is not empty + if [ -n "$volumeName" ]; then + # Add the volume path to the volume paths string + volumePaths+=" /var/lib/docker/volumes/$volumeName" + fi + done +done + +# Calculate the total size of the volumes +# shellcheck disable=SC2086 +totalSize=$(du -csh $volumePaths 2>/dev/null | grep total | awk '{print $1}') + +# Print the total size of the volumes +echo "✅ Total size of volumes to migrate: $totalSize" + +# Print size of backupSourceDir +backupSourceDirSize=$(du -csh $backupSourceDir 2>/dev/null | grep total | awk '{print $1}') +echo "✅ Size of the source directory: $backupSourceDirSize" + +# Check if the backup file already exists +if [ ! -f "$backupFileName" ]; then + echo "🚸 Backup file does not exist, creating" + + # Recommend stopping docker before creating the backup + echo "🚸 It's recommended to stop all Docker containers before creating the backup + Do you want to stop Docker? (y/n)" + read -r answer + if [ "$answer" != "${answer#[Yy]}" ]; then + if ! systemctl stop docker; then + echo "❌ Docker stop failed" + exit 1 + fi + echo "✅ Docker stopped" + else + echo "🚸 Docker not stopped, continuing with the backup" + fi + + # shellcheck disable=SC2086 + if ! tar --exclude='*.sock' -Pczf $backupFileName -C / $backupSourceDir $HOME/.ssh/authorized_keys $volumePaths; then + echo "❌ Backup file creation failed" + exit 1 + fi + echo "✅ Backup file created" +else + echo "🚸 Backup file already exists, skipping creation" +fi + +# Define the remote commands to be executed +remoteCommands=" + # Check if Docker is a service + if systemctl is-active --quiet docker; then + # Stop Docker if it's a service + if ! systemctl stop docker; then + echo '❌ Docker stop failed'; + exit 1; + fi + echo '✅ Docker stopped'; + else + echo 'ℹ️ Docker is not a service, skipping stop command'; + fi + + echo '🚸 Saving existing authorized keys...'; + cp ~/.ssh/authorized_keys ~/.ssh/authorized_keys_backup; + + echo '🚸 Extracting backup file...'; + if ! tar -Pxzf - -C /; then + echo '❌ Backup file extraction failed'; + exit 1; + fi + echo '✅ Backup file extracted'; + + echo '🚸 Merging authorized keys...'; + cat ~/.ssh/authorized_keys_backup ~/.ssh/authorized_keys | sort | uniq > ~/.ssh/authorized_keys_temp; + mv ~/.ssh/authorized_keys_temp ~/.ssh/authorized_keys; + chmod 600 ~/.ssh/authorized_keys; + echo '✅ Authorized keys merged'; + + if ! curl -fsSL https://cdn.coollabs.io/coolify/install.sh | bash; then + echo '❌ Coolify installation failed'; + exit 1; + fi + echo '✅ Coolify installed'; +" + +# SSH to the destination server, execute the remote commands +if ! ssh -i "$sshKeyPath" -o "StrictHostKeyChecking no" root@$destinationHost "$remoteCommands" <$backupFileName; then + echo "❌ Remote commands execution or Docker restart failed" + exit 1 +fi +echo "✅ Remote commands executed successfully" + +# Clean up - Ask the user for confirmation before removing the local backup file +echo "Do you want to remove the local backup file? (y/n)" +read -r answer +if [ "$answer" != "${answer#[Yy]}" ]; then + if ! rm -f $backupFileName; then + echo "❌ Failed to remove local backup file" + exit 1 + fi + echo "✅ Local backup file removed" +else + echo "🚸 Local backup file not removed" +fi diff --git a/seeker/snippet/qt5-build-for-qgis-macos.sh b/seeker/snippet/qt5-build-for-qgis-macos.sh new file mode 100644 index 00000000..bce80b01 --- /dev/null +++ b/seeker/snippet/qt5-build-for-qgis-macos.sh @@ -0,0 +1,136 @@ +#date: 2024-07-09T16:58:15Z +#url: https://api.github.com/gists/1716e2bc5e4b3a6ac158bf2fd75c42eb +#owner: https://api.github.com/users/smellman + +#!/bin/bash + +# This script is used to build Qt5 for QGIS on macOS. +# see: https://github.com/Homebrew/homebrew-core/blob/HEAD/Formula/q/qt@5.rb + +# Set the version of Qt5 to build +QT_VERSION=5.15.14 +SRC_DIR=qt-everywhere-src-${QT_VERSION} +TAR_BALL=qt-everywhere-opensource-src-${QT_VERSION}.tar.xz +PREFIX=/opt/qt/${QT_VERSION} + +# unarchive the tarball + +tar jxf ${TAR_BALL} + +# Change directory to the source directory + +cd ${SRC_DIR} + +# Configure qtwebengine start +## git clone qtwebengine + +rm -fr qtwebengine +git clone https://code.qt.io/qt/qtwebengine.git qtwebengine +cd qtwebengine +git checkout -b v5.15.16-lts refs/tags/v5.15.16-lts +git submodule update --init --recursive + +## patch qtwebengine +### libxml2 2.12 compatibility +cd src/3rdparty +curl "https://github.com/qt/qtwebengine-chromium/commit/c98d28f2f0f23721b080c74bc1329871a529efd8.patch?full_index=1" | patch -p 1 +cd ../.. + +### python3 support for qtwebengine-chromium + +cd src/3rdparty +curl "https://raw.githubusercontent.com/Homebrew/formula-patches/7ae178a617d1e0eceb742557e63721af949bd28a/qt5/qt5-webengine-chromium-python3.patch?full_index=1" | patch -p 1 +cd ../.. + +### python3 support for qtwebengine + +curl "https://raw.githubusercontent.com/Homebrew/formula-patches/a6f16c6daea3b5a1f7bc9f175d1645922c131563/qt5/qt5-webengine-python3.patch?full_index=1" | patch -p 1 + +### Use Debian patch to support Python 3.11 + +curl "https://raw.githubusercontent.com/Homebrew/formula-patches/9758d3dd8ace5aaa9d6720b2e2e8ea1b863658d5/qt5/qtwebengine-py3.11.patch" | patch -p 1 + +### Fix ffmpeg build with binutils + +cd src/3rdparty/chromium/third_party/ffmpeg +curl "https://github.com/FFmpeg/FFmpeg/commit/effadce6c756247ea8bae32dc13bb3e6f464f0eb.patch?full_index=1" | patch -p 1 +cd ../../../../.. + +### Use Gentoo's patch for ICU 74 support + +curl "https://gitweb.gentoo.org/repo/gentoo.git/plain/dev-qt/qtwebengine/files/qtwebengine-6.5.3-icu74.patch?id=ba397fa71f9bc9a074d9c65b63759e0145bb9fa0" | patch -p 1 + +## patch qtwebengine end +cd .. + +# configure qtwebengine end + +# cataplut +git clone https://chromium.googlesource.com/catapult.git +cd catapult +git checkout 5eedfe23148a234211ba477f76fc2ea2e8529189 +cd .. + +# Fix build with Xcode 14.3. + +cd qtlocation/src/3rdparty/mapbox-gl-native +curl "https://invent.kde.org/qt/qt/qtlocation-mapboxgl/-/commit/5a07e1967dcc925d9def47accadae991436b9686.diff" | patch -p 1 +cd ../../../.. + +# Fix qmake with Xcode 15. + +curl "https://raw.githubusercontent.com/Homebrew/formula-patches/086e8cf/qt5/qt5-qmake-xcode15.patch" | patch -p 1 + +# Fix qtmultimedia build with Xcode 15 + +curl "https://raw.githubusercontent.com/Homebrew/formula-patches/3f509180/qt5/qt5-qtmultimedia-xcode15.patch" | patch -p 1 + +# Fix use of macOS 14 only memory_resource on macOS 13 + +cd qtbase +curl "https://raw.githubusercontent.com/macports/macports-ports/56a9af76a6bcecc3d12c3a65f2465c25e05f2559/aqua/qt5/files/patch-qtbase-memory_resource.diff" | patch -p 0 +cd .. + +# CVE-2023-32763 Remove with Qt 5.15.15 + +cd qtbase +curl "https://invent.kde.org/qt/qt/qtbase/-/commit/deb7b7b52b6e6912ff8c78bc0217cda9e36c4bba.diff" | patch -p 1 +cd .. + +# CVE-2023-34410 Remove with Qt 5.15.15 + +cd qtbase +curl "https://invent.kde.org/qt/qt/qtbase/-/commit/2ad1884fee697e0cb2377f3844fc298207e810cc.diff" | patch -p 1 +cd .. + +# CVE-2023-37369 Remove with Qt 5.15.15 + +cd qtbase +curl "https://ftp.yz.yamagata-u.ac.jp/pub/qtproject/archive/qt/5.15/CVE-2023-37369-qtbase-5.15.diff" | patch -p 1 +cd .. + +# CVE-2023-38197 Remove with Qt 5.15.15 + +cd qtbase +curl "https://ftp.yz.yamagata-u.ac.jp/pub/qtproject/archive/qt/5.15/CVE-2023-38197-qtbase-5.15.diff" | patch -p 1 +cd .. + +# CVE-2023-51714 Remove with Qt 5.15.17 + +cd qtbase +curl "https://download.qt.io/official_releases/qt/5.15/0001-CVE-2023-51714-qtbase-5.15.diff" | patch -p 1 +curl "https://download.qt.io/official_releases/qt/5.15/0002-CVE-2023-51714-qtbase-5.15.diff" | patch -p 1 +cd .. + + +# patches end + +# move catapult to qtwebengine +rm -fr qtwebengine/src/3rdparty/chromium/third_party/catapult +mv catapult qtwebengine/src/3rdparty/chromium/third_party/ + +# Configure the build + +./configure -verbose -prefix ${PREFIX} -release -opensource -confirm-license -nomake examples -nomake tests -pkg-config -dbus-runtime -proprietary-codecs -no-rpath +make -j$(sysctl -n hw.logicalcpu) +make install diff --git a/seeker/snippet/solution.go b/seeker/snippet/solution.go new file mode 100644 index 00000000..7d340fd6 --- /dev/null +++ b/seeker/snippet/solution.go @@ -0,0 +1,122 @@ +//date: 2024-07-09T16:53:54Z +//url: https://api.github.com/gists/ec88f23ff4011b784ad92a7406529876 +//owner: https://api.github.com/users/elliotcubit + +package main + +import ( + "bytes" + "fmt" +) + +const Flag = "xxxxxxxx-xxxx-Mxxx-Nxxx-xxxxxxxxxxxx" +const UUIDv4StringLen = len(Flag) +const FlagStringLen = UUIDv4StringLen + 6 + +func SearchString(mem *Memory) (string, error) { + p := 0 + for { + page, err := mem.ReadPage(p) + if err != nil { + return "", err + } + + // Check for the full prefix + idx := bytes.Index(page, []byte("gc24{")) + if idx != -1 { + s, err := flagAtIdx(mem, p*len(page)+idx) + if err != nil { + return "", err + } + + if s != "" { + fmt.Println(s) + return s, nil + } + } + + // Or these prefixes, at the end of the page + idx = bytes.LastIndex(page, []byte("gc24")) + if idx == len(page)-4 { + s, err := flagAtIdx(mem, p*len(page)+idx) + if err != nil { + return "", err + } + + if s != "" { + fmt.Println(s) + return s, nil + } + } + + idx = bytes.LastIndex(page, []byte("gc2")) + if idx == len(page)-3 { + s, err := flagAtIdx(mem, p*len(page)+idx) + if err != nil { + return "", err + } + + if s != "" { + fmt.Println(s) + return s, nil + } + } + + idx = bytes.LastIndex(page, []byte("gc")) + if idx == len(page)-2 { + s, err := flagAtIdx(mem, p*len(page)+idx) + if err != nil { + return "", err + } + + if s != "" { + fmt.Println(s) + return s, nil + } + } + + idx = bytes.LastIndexByte(page, 'g') + if idx == len(page)-1 { + s, err := flagAtIdx(mem, p*len(page)+idx) + if err != nil { + return s, nil + } + + if s != "" { + fmt.Println(s) + return s, nil + } + } + + p++ + + if p > 100000 { + return "", fmt.Errorf("sanity check. have a nice day.") + } + } +} + +func flagAtIdx(mem *Memory, idx int) (string, error) { + end, err := mem.ReadAddress(idx + FlagStringLen - 1) + if err != nil { + return "", fmt.Errorf("check flag: %w", err) + } + + if end != '}' { + return "", nil + } + + var b bytes.Buffer + + b.WriteString("gc24{") + + for i := idx + 5; i < idx+FlagStringLen; i++ { + bb, err := mem.ReadAddress(i) + if err != nil { + return "", fmt.Errorf("check flag: %w", err) + } + _ = b.WriteByte(bb) + } + + return string(b.Bytes()), nil +} diff --git a/seeker/snippet/transformer.py b/seeker/snippet/transformer.py new file mode 100644 index 00000000..f2bae4b9 --- /dev/null +++ b/seeker/snippet/transformer.py @@ -0,0 +1,214 @@ +#date: 2024-07-09T17:12:31Z +#url: https://api.github.com/gists/55063a19cf0243bc521bd854185226c0 +#owner: https://api.github.com/users/SF9040 + +""" +The 2024 Transformer (the Noam Transformer): +- RMSNorm +- GQA or some combination +- Sliding window attention +- Swiglu +- RoPE (Rotary Positional Embedding) + +LLM Arches: + hidden | MLP mult. | n_layers | rope_theta | GQA Group Size | GLU Act. | ops + +Gemma 2 9B 3584 | 4x | 42 | 10000 | 2 | GELU Tanh | norm -> attn -> norm -> add -> norm -> mlp -> norm -> add +Llama 3 8B 4096 | 3.5x | 32 | 50000 | 4 | SILU | norm -> attn -> add -> norm -> mlp -> add +Mistral 7B 4096 | 3.5x | 32 | 1000000 | 4 | SILU | norm -> attn -> add -> norm -> mlp -> add +Qwen 2 7B 3584 | ~5.29x | 28 | 1000000 | 7 | SILU | norm -> attn -> add -> norm -> mlp -> add +InternLM2.5 7B 4096 | 3.5x | 32 | 50000000 | 4 | SILU | norm -> attn -> add -> norm -> mlp -> add +DeepSeek 6.7B 4096 | 2.6875x | 32 | 100000 | 1 | SILU | norm -> attn -> add -> norm -> mlp -> add + +Phi 3 14B | 4096 | 4.375x | 40 | 10000 | 4 | SILU | norm -> attn -> (drop) add -> norm -> mlp -> (drop) add + +Gemma 2 27| 4608 | 8x | 46 | 10000 | 2 | GELU Tanh | norm -> attn -> norm -> add -> norm -> mlp -> norm -> add +DeepSeek 33B 7168 | ~2.68x | 62 | 100000 | 7 | SILU | norm -> attn -> add -> norm -> mlp -> add +Llama 3 70B 8192 | 3.5x | 80 | 50000 | 8 | SILU | norm -> attn -> add -> norm -> mlp -> add +Qwen 2 72B 8192 | ~3.61x | 80 | 1000000 | 8 | SILU | norm -> attn -> add -> norm -> mlp -> add + +Others: +- Gemma 2 uses logit softcapping (50), query pre attention scaling + +References: +- https://github.com/naklecha/llama3-from-scratch/tree/main +- https://github.com/xjdr-alt/simple_transformer +- https://github.com/google/gemma_pytorch/tree/main +- https://github.com/hkproj/pytorch-llama/tree/main +""" + +import torch +import torch.nn.functional as F +from typing import List, NamedTuple + + +NUM_Q_HEADS = 32 # Llama numbers +NUM_KV_HEADS = 8 # Llama numbers +SLIDING_WINDOW_SIZE = 4096 + + +class LayerWeights(NamedTuple): + input_norm: torch.Tensor # (hidden) + post_attn_norm: torch.Tensor # (hidden) + q_proj: torch.Tensor # (hidden, q_intermediate) + k_proj: torch.Tensor # (hidden, kv_intermediate) + v_proj: torch.Tensor # (hidden, kv_intermediate) + o_proj: torch.Tensor # (q_intermediate, hidden) + gate_proj: torch.Tensor # (hidden, intermediate) + up_proj: torch.Tensor # (hidden, intermediate) + down_proj: torch.Tensor # (intermediate, hidden) + + +class TransformerWeights(NamedTuple): + layers: List[LayerWeights] + token_emb: "**********" + final_norm: torch.Tensor # (hidden) + lm_head: torch.Tensor # (hidden, vocab_size) + + +def norm(x: torch.Tensor, weight: torch.Tensor): + in_dtype = x.dtype + x = x.float() + out = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + 1e-5) # eps might change depending on the model + return weight * out.to(in_dtype) + + +def ffn(x: torch.Tensor, weights: LayerWeights): + gate = F.silu(x @ weights.gate_proj) + fused = gate * (x @ weights.up_proj) + return fused @ weights.down_proj + + +def rope(x: torch.Tensor, freqs_cis: torch.Tensor): + def rotate(x): + """ + rotate_half(torch.arange(4)) + > tensor([-2, -3, 0, 1]) + """ + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + cos, sin = freqs_cis + cos, sin = cos.type_as(x), sin.type_as(x) + right = rotate(x.reshape(*x.shape[:-1], -1, 2)).reshape(x.shape) + out = x * cos + right * sin + return out.to(x.dtype) + + +def attn( + x: torch.Tensor, + weights: LayerWeights, + freqs_cis: tuple, + sliding_window_size=None, +): + bs, seq_len, d_model = x.shape + xq, xk, xv = x @ weights.q_proj, x @ weights.k_proj, x @ weights.v_proj + xq = xq.view(bs, seq_len, NUM_Q_HEADS, -1).transpose(1, 2) # (bs, NUM_Q_HEADS, seq_len, q_intermediate) + xk = xk.view(bs, seq_len, NUM_KV_HEADS, -1).transpose(1, 2) # (bs, NUM_KV_HEADS, seq_len, kv_intermediate) + xv = xv.view(bs, seq_len, NUM_KV_HEADS, -1).transpose(1, 2) # (bs, NUM_KV_HEADS, seq_len, kv_intermediate) + head_dim = xq.shape[-1] + + # Treat GQA as MHA and just repeat along the head dimension + xk = torch.repeat_interleave(xk, NUM_Q_HEADS // NUM_KV_HEADS, dim=1) + xv = torch.repeat_interleave(xv, NUM_Q_HEADS // NUM_KV_HEADS, dim=1) + xq = rope(xq, freqs_cis) + xk = rope(xk, freqs_cis) + + attn_scores = (xq @ xk.transpose(2, 3)) * (head_dim**-0.5) + mask = torch.triu(torch.full((bs, seq_len, seq_len), -2.3819763e38), diagonal=1) # This number is taken from Gemma + if sliding_window_size is not None: # Sliding window attention + all_ones = torch.ones((seq_len, seq_len)) + sliding_mask = torch.triu(all_ones, -1 * sliding_window_size + 1) * torch.tril(all_ones, sliding_window_size - 1) + mask = torch.where(sliding_mask == 1, mask, -2.3819763e38) + mask = mask.to(x.device, x.dtype) + attn_scores = attn_scores + mask + attn_probs = F.softmax(attn_scores, dim=-1) + attn_out = attn_probs @ xv + attn_out = attn_out.transpose(1, 2).contiguous().view(bs, seq_len, -1) + return attn_out @ weights.o_proj + + +# for efficiency, should precompute for 0..max_length * 2 then select [:curr_length] +def precompute_freqs_cis(head_dim: int, seq_len: int, base_theta: float = 500000.0): + inv_freqs = 1.0 / (base_theta ** (torch.arange(0, head_dim, 2).float() / head_dim)) # Eq 15: theta_{1} ... theta_{dim/2}. Shape: (dim/2) + m = torch.arange(seq_len) # all possible position indices + freqs = torch.outer(m, inv_freqs).float() # [m_i * theta_j] for all i (positions) and j (frequencies). Shape: (seq_len, dim/2) | freqs[i][j] == m[i] * inv_freqs[j] + cos = torch.cos(freqs) # Shape: (seq_len, dim/2) + cos = torch.repeat_interleave(cos, 2, dim=-1) # Shape: (seq_len, dim) + sin = torch.sin(freqs) # Shape: (seq_len, dim/2) + sin = torch.repeat_interleave(sin, 2, dim=-1) # Shape: (seq_len, dim) + return (cos, sin) + + +def transformer(in_tokens: "**********": TransformerWeights): + x = "**********" + b, t, d = x.shape + q_intermediate = weights.layers[0].q_proj.shape[1] + freqs_cis = precompute_freqs_cis(q_intermediate // NUM_Q_HEADS, t) # (cos, sin) + for i, layer in enumerate(weights.layers): + residual = x + hidden = norm(x, layer.input_norm) + hidden = attn(hidden, layer, freqs_cis, sliding_window_size=SLIDING_WINDOW_SIZE if i % 6 != 0 else None) # Follows https://research.character.ai/optimizing-inference/ + hidden = residual + hidden + + residual = hidden + hidden = norm(hidden, layer.post_attn_norm) + hidden = ffn(hidden, layer) + hidden = residual + hidden + x = hidden + + x = norm(x, weights.final_norm) + x = x @ weights.lm_head + return x + + +if __name__ == "__main__": + from transformers import AutoTokenizer, AutoModelForCausalLM + + tokenizer = "**********" + # Download the official repo weights + state_dict = torch.load("Meta-Llama-3-8B/consolidated.00.pth", map_location="cuda") + layers = [] + n_layers = 32 + for i in range(n_layers): + layer = LayerWeights( + input_norm=state_dict[f"layers.{i}.attention_norm.weight"], + post_attn_norm=state_dict[f"layers.{i}.ffn_norm.weight"], + q_proj=state_dict[f"layers.{i}.attention.wq.weight"].t(), + k_proj=state_dict[f"layers.{i}.attention.wk.weight"].t(), + v_proj=state_dict[f"layers.{i}.attention.wv.weight"].t(), + o_proj=state_dict[f"layers.{i}.attention.wo.weight"].t(), + gate_proj=state_dict[f"layers.{i}.feed_forward.w1.weight"].t(), + up_proj=state_dict[f"layers.{i}.feed_forward.w3.weight"].t(), + down_proj=state_dict[f"layers.{i}.feed_forward.w2.weight"].t(), + ) + layers.append(layer) + + weights = TransformerWeights( + layers=layers, + token_emb= "**********" + final_norm=state_dict["norm.weight"], + lm_head=state_dict["output.weight"].t(), + ) + + prompt = "the answer to the ultimate question of life " + in_tokens = "**********"="pt")["input_ids"].to("cuda") + for _ in range(10): + out = "**********" + next_token = torch.argmax(out[: "**********":]) + in_tokens = "**********"=1) + + del weights + del state_dict + + model = AutoModelForCausalLM.from_pretrained("meta-llama/Meta-Llama-3-8B", torch_dtype=torch.bfloat16, device_map="auto", _attn_implementation="eager", use_cache=False) + input_ids = "**********"="pt").to("cuda") + outputs = "**********"=10, num_beams=1, do_sample=False) + print("Ours: "**********" + print("Ref: "**********" +ementation="eager", use_cache=False) + input_ids = tokenizer(prompt, return_tensors="pt").to("cuda") + outputs = model.generate(**input_ids, max_new_tokens=10, num_beams=1, do_sample=False) + print("Ours:", tokenizer.decode(in_tokens[0].tolist())) + print("Ref:", tokenizer.decode(outputs[0])) diff --git a/seeker/snippet/uuid7.py b/seeker/snippet/uuid7.py new file mode 100644 index 00000000..346cdcc9 --- /dev/null +++ b/seeker/snippet/uuid7.py @@ -0,0 +1,31 @@ +#date: 2024-07-09T17:08:31Z +#url: https://api.github.com/gists/ac520655d36b746c5cf22bf9c06b9811 +#owner: https://api.github.com/users/ericbuckley + +import os +import time + +def uuidv7(): + # random bytes + value = bytearray(os.urandom(16)) + + # current timestamp in ms + timestamp = int(time.time() * 1000) + + # timestamp + value[0] = (timestamp >> 40) & 0xFF + value[1] = (timestamp >> 32) & 0xFF + value[2] = (timestamp >> 24) & 0xFF + value[3] = (timestamp >> 16) & 0xFF + value[4] = (timestamp >> 8) & 0xFF + value[5] = timestamp & 0xFF + + # version and variant + value[6] = (value[6] & 0x0F) | 0x70 + value[8] = (value[8] & 0x3F) | 0x80 + + return value + +if __name__ == "__main__": + uuid_val = uuidv7() + print(''.join(f'{byte:02x}' for byte in uuid_val)) \ No newline at end of file