2024-09-03 17:13:03.454120 new snippets

seeker/report.txt

@@ -1,3 +1,25 @@
+--------------------------------------------------------------------------------
+ 2024-09-03 17:13:03.454120
+--------------------------------------------------------------------------------
+  On branch main
+Your branch is up to date with 'origin/main'.
+
+Untracked files:
+  (use "git add <file>..." to include in what will be committed)
+	snippet/ST_Point_table_from_csv.py
+	snippet/ascii-smuggler.py
+	snippet/bump
+	snippet/context_builder.py
+	snippet/dd.java
+	snippet/function.sh
+	snippet/instala_ccd_serpro_ubuntu.sh
+	snippet/login_to_google.py
+	snippet/polygon_table_from_datafile.py
+	snippet/sedona_df_to_geopandas.py
+	snippet/spatial_joins.py
+
+nothing added to commit but untracked files present (use "git add" to track)
+
 --------------------------------------------------------------------------------
  2024-09-02 17:11:30.186703
 --------------------------------------------------------------------------------

seeker/snippet/ST_Point_table_from_csv.py

@@ -0,0 +1,19 @@
+#date: 2024-09-03T16:56:27Z
+#url: https://api.github.com/gists/4e27bb8bae3bbff729866e1bc5866f8c
+#owner: https://api.github.com/users/SeanLikesData
+
+point_csv_df = sedona.read.format("csv").\
+    option("delimiter", ",").\
+    option("header", "false").\
+    load("data/testpoint.csv")
+
+point_csv_df.createOrReplaceTempView("pointtable")
+
+point_df = sedona.sql("
+                        select ST_Point(
+                            cast(pointtable._c0 as Decimal(24,20)), 
+                            cast(pointtable._c1 as Decimal(24,20))
+                            ) 
+                        as arealandmark from pointtable
+                    ")
+point_df.show(5)

seeker/snippet/ascii-smuggler.py

@@ -0,0 +1,22 @@
+#date: 2024-09-03T16:59:13Z
+#url: https://api.github.com/gists/00507d3271b22c698a83e99990560bf8
+#owner: https://api.github.com/users/rossja
+
+# smuggle text hidden as invisible unicode chars
+# credit to jthacker: https://x.com/rez0__/status/1745545813512663203
+# and embrace the red: https://embracethered.com/blog/posts/2024/hiding-and-finding-text-with-unicode-tags/
+
+import pyperclip
+
+def convert_to_tag_chars(input_string):
+  return ''.join(chr(0xE0000 + ord(ch)) for ch in input_string)
+
+# Example usage:
+
+user_input = input("Enter a string to convert to tag characters: ")
+tagged_output = convert_to_tag_chars(user_input)
+
+print(f"Tagged output:\n")
+print(f"START{tagged_output}END")
+
+pyperclip.copy(tagged_output)

seeker/snippet/bump

@@ -0,0 +1,27 @@
+#date: 2024-09-03T17:12:01Z
+#url: https://api.github.com/gists/2c8108dfaafccc0553fef195b88efe70
+#owner: https://api.github.com/users/garrettdimon
+
+#!/usr/bin/env bash
+
+# Exit on error
+set -o errexit
+
+# Get the version from the .ruby-version file
+target_ruby_version=$(cat .ruby-version)
+
+# Make sure the latest Ruby versions are present and available
+arch -arm64 brew update && arch -arm64 brew upgrade ruby-build
+
+# List latest stable versions for each Ruby for verification
+rbenv install --list
+
+# Install the version specified in the .ruby-version file
+rbenv install $target_ruby_version
+
+# Wipe out the Gemfile.lock file
+rm Gemfile.lock
+
+# Re-bundle and ensure linux is added
+bundle install
+bundle lock --add-platform x86_64-linux

seeker/snippet/context_builder.py

@@ -0,0 +1,12 @@
+#date: 2024-09-03T16:56:27Z
+#url: https://api.github.com/gists/4e27bb8bae3bbff729866e1bc5866f8c
+#owner: https://api.github.com/users/SeanLikesData
+
+config = SedonaContext.builder() .\
+    config('spark.jars.packages',
+           'org.apache.sedona:sedona-spark-3.4_2.12:1.6.0,'
+           'org.datasyslab:geotools-wrapper:1.6.0-28.2,'
+           'uk.co.gresearch.spark:spark-extension_2.12:2.11.0-3.4'). \
+    getOrCreate()
+
+sedona = SedonaContext.create(config)

seeker/snippet/dd.java

@@ -0,0 +1,60 @@
+//date: 2024-09-03T17:01:38Z
+//url: https://api.github.com/gists/28083a29785c2f6a3ec5816a8320a739
+//owner: https://api.github.com/users/jorgecuza92
+
+package com.capitalone.engagementexperience.reststepdefinitions.directdeposit;
+
+import com.capitalone.banxoa.cucumber.AbstractRESTStepDefinition;
+import com.capitalone.engagementexperience.model.ScenarioData;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.concurrent.TimeUnit;
+
+public class DirectDepositStepDefinitions extends AbstractRESTStepDefinition {
+
+    // Autowired to inject ScenarioData, which contains transaction data
+    @Autowired
+    private ScenarioData scenarioData;
+
+    // Method to set the enrollment status of Direct Deposit based on input and transaction time
+    @And("^the account IsDirectDepositEnrolled value is (string)$")
+    public void setScenarioDataToSupportDirectDepositEnrolment(String isDirectDepositEnrolledString) throws Exception {
+        // Parse the input string to determine if direct deposit is enrolled
+        boolean isDirectDepositEnrolled = Boolean.parseBoolean(isDirectDepositEnrolledString);
+
+        // Fetch the last transaction time; replace with your method to get the correct date string if necessary
+        String lastTransactionTime = scenarioData.getLastTransactionTime(); // Ensure this fetches the correct timestamp
+
+        // Convert the lastTransactionTime from String to Date object
+        DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); // Adjust format if necessary
+        Date transactionDate = dateFormat.parse(lastTransactionTime);
+        Date currentDate = new Date(); // Get the current date and time
+
+        // Calculate the difference in days between the current date and the last transaction date
+        long differenceInMillis = currentDate.getTime() - transactionDate.getTime();
+        long daysDifference = TimeUnit.MILLISECONDS.toDays(differenceInMillis);
+
+        // Logic to determine if the enrollment status needs to be updated
+        if (isDirectDepositEnrolled && daysDifference > 15) {
+            // Update status to unenrolled if last transaction is older than 15 days
+            scenarioData.setClientCorrelationId("unenrolled");
+            scenarioData.setCustomerReferenceId("Direct Deposit inactive for over 15 days");
+            System.out.println("Updated: Direct Deposit is not currently enrolled due to inactivity.");
+        } else {
+            // Keep enrollment as active if within 15 days
+            scenarioData.setClientCorrelationId("enrolled");
+            scenarioData.setCustomerReferenceId("Direct Deposit active");
+        }
+    }
+
+    // Example of another existing method for reference, no changes needed
+    @And("^invalid (string) is provided in scenario$")
+    public void setInvalidFieldInScenarioData(String fieldName) {
+        if (!fieldName.equals("CustomerReferenceId")) {
+            scenarioData.setClientCorrelationId("invalid");
+        }
+    }
+}

seeker/snippet/function.sh

@@ -0,0 +1,30 @@
+#date: 2024-09-03T16:48:38Z
+#url: https://api.github.com/gists/efda108268ac08bb45984f8bacf965aa
+#owner: https://api.github.com/users/mkpanq
+
+# Function to set the latest version of a package locally and install if necessary
+asdf_latest_local() {
+  local plugin="$1"
+  local latest_version
+
+  # Check if a plugin name is provided
+  if [[ -z "$plugin" ]]; then
+    echo "Usage: asdf_latest_local <plugin>"
+    return 1
+  fi
+
+  # Get the latest version available for the plugin
+  latest_version=$(asdf latest "$plugin")
+
+  # Check if the latest version is already installed
+  if ! asdf list "$plugin" | grep -q "$latest_version"; then
+    echo "Installing latest version of $plugin: $latest_version"
+    asdf install "$plugin" "$latest_version"
+  else
+    echo "Latest version of $plugin is already installed: $latest_version"
+  fi
+
+  # Set the latest version locally
+  echo "Setting $plugin to use version $latest_version locally"
+  asdf local "$plugin" "$latest_version"
+}

seeker/snippet/instala_ccd_serpro_ubuntu.sh

@@ -0,0 +1,40 @@
+#date: 2024-09-03T16:48:17Z
+#url: https://api.github.com/gists/2b409571e3939324a53e30c8ffad8b5d
+#owner: https://api.github.com/users/rodrigomaia
+
+# Entrar na pasta de certificados do sistema:
+# Ubuntu anterior ao 18:
+sudo mkdir /usr/share/ca-certificates/serpro/
+sudo cd /usr/share/ca-certificates/serpro/
+# Ubuntu 18:
+sudo mkdir /usr/local/share/ca-certificates/serpro/
+cd /usr/local/share/ca-certificates/serpro/
+
+# Baixar os certificados do repositorio:
+wget -r --no-check-certificate https://ccd.serpro.gov.br/serproacf/docs/
+
+# Remover apenas os certificados de interesse:
+find ./ccd.serpro.gov.br -name *.crt | sudo xargs -I{} cp -u {} .
+
+# Limpar o restante do wget:
+sudo rm -rf ccd.serpro.gov.br/
+
+# Executar compilação dos certificados para o sistema:
+sudo update-ca-certificates
+
+# Instalar os certificados nos browsers
+sudo apt install libnss3-tools
+
+# Instalar os certificados no google-chrome
+for i in $(ls /usr/local/share/ca-certificates/serpro/); do $(certutil -d sql:$HOME/.pki/nssdb -A -t "C,C,C" -n $i -i /usr/local/share/ca-certificates/serpro/$i); done
+
+# Instalar os certificados no firefox
+cat ~/.mozilla/firefox/profiles.ini 
+
+# Anote o valor do Default no meu caso: yxsfy966.default-release
+# [Install4F96D1932A9F858E]
+# Default=yxsfy966.default-release
+# Locked=1
+
+# O Resultado você subistitui no comando abaixo logo depois do $HOME/.mozilla/firefox/:    
+for i in $(ls /usr/local/share/ca-certificates/serpro/); do $(certutil -d sql:$HOME/.mozilla/firefox/yxsfy966.default-release -A -t "C,C,C" -n $i -i /usr/local/share/ca-certificates/serpro/$i); done

seeker/snippet/login_to_google.py

@@ -0,0 +1,97 @@
+#date: 2024-09-03T17:08:18Z
+#url: https://api.github.com/gists/7e30a1630f9c2d59d619b4b7e4008e2e
+#owner: https://api.github.com/users/samuelvillegas
+
+"""
+The most important part is replacing the word `headless` from the user Agent
+"""
+import selenium.webdriver.chrome.webdriver
+import selenium.webdriver.chrome.options
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.common.keys import Keys
+import time
+
+email = '[email protected]'
+password = "**********"
+
+# Setting up Chrome options
+driver_opts = selenium.webdriver.chrome.options.Options()
+driver_opts.add_argument('--no-sandbox')
+driver_opts.add_argument('--disable-dev-shm-usage')
+
+# Disable GPU hardware acceleration
+# To prevent gpu related issues
+driver_opts.add_argument('--disable-gpu')
+
+driver_opts.add_argument('--window-size=1920,1080')
+driver_opts.add_argument('--start-maximized')
+
+# Disable the Blink feature that detects automation
+driver_opts.add_argument('--disable-blink-features=AutomationControlled')
+
+# Exclude the 'enable-automation' switch to make automation less detectable
+driver_opts.add_experimental_option('excludeSwitches', ['enable-automation'])
+
+# Disable the use of the automation extension
+driver_opts.add_experimental_option('useAutomationExtension', False)
+
+# driver_opts.add_argument('--incognito')  # Open the browser in incognito mode
+
+# Run the browser in headless mode
+driver_opts.add_argument('--headless')
+
+# Initialize Chrome WebDriver
+driver = selenium.webdriver.chrome.webdriver.WebDriver(options=driver_opts)
+
+# Remove 'Headless' from the user agent
+# So, websites can't detect that the browser is running in headless mode
+current_user_agent = driver.execute_script("return navigator.userAgent;")
+driver.execute_cdp_cmd(
+    'Network.setUserAgentOverride',
+    {
+        "userAgent": current_user_agent.replace('Headless', ''),
+    },
+)
+
+# Prevent websites from detecting that the browser is being controlled by automation tools
+driver.execute_cdp_cmd('Page.addScriptToEvaluateOnNewDocument', {
+    'source': '''
+        Object.defineProperty(navigator, 'webdriver', {
+          get: () => undefined
+        });
+    '''
+})
+
+try:
+    # Open Google login page
+    driver.get('https://accounts.google.com/signin')
+
+    # Enter email or phone
+    email_input = WebDriverWait(driver, 10).until(
+        EC.presence_of_element_located((By.ID, 'identifierId'))
+    )
+    time.sleep(2)  # Small delay
+    email_input.send_keys(email)
+    email_input.send_keys(Keys.RETURN)
+
+    # Wait for password input to appear and enter password
+    password_input = "**********"
+        EC.presence_of_element_located((By.NAME, 'Passwd'))
+    )
+    time.sleep(2)  # Small delay
+    password_input.send_keys(password)
+    password_input.send_keys(Keys.RETURN)
+
+    # Wait for successful login (Example: Google home page)
+    WebDriverWait(driver, 10).until(
+        EC.title_contains('Google'),
+    )
+    print("Login successful!")
+
+finally:
+    # Close the browser
+    driver.quit()
+ser
+    driver.quit()

seeker/snippet/polygon_table_from_datafile.py

@@ -0,0 +1,36 @@
+#date: 2024-09-03T16:56:27Z
+#url: https://api.github.com/gists/4e27bb8bae3bbff729866e1bc5866f8c
+#owner: https://api.github.com/users/SeanLikesData
+
+# code snippets for generating polygon tables by reading various file types.
+# Note that the only real change is the ST_GeomFrom{filetype} function in line 7.
+
+# ST_GeomfromText
+polygon_wkt_df = sedona.read.format("csv").\
+    option("delimiter", "\t").\
+    option("header", "false").\
+    load("data/county_small.tsv")
+
+polygon_wkt_df.createOrReplaceTempView("polygontable")
+polygon_df = sedona.sql("select polygontable._c6 as name, ST_GeomFromText(polygontable._c0) as countyshape from polygontable")
+polygon_df.show(5)
+
+# ST_GeomFromWKB
+polygon_wkb_df = sedona.read.format("csv").\
+    option("delimiter", "\t").\
+    option("header", "false").\
+    load("data/county_small_wkb.tsv")
+
+polygon_wkb_df.createOrReplaceTempView("polygontable")
+polygon_df = sedona.sql("select polygontable._c6 as name, ST_GeomFromWKB(polygontable._c0) as countyshape from polygontable")
+polygon_df.show(5)
+
+# ST_GeomFromGeoJSON
+polygon_json_df = sedona.read.format("csv").\
+    option("delimiter", "\t").\
+    option("header", "false").\
+    load("data/testPolygon.json")
+
+polygon_json_df.createOrReplaceTempView("polygontable")
+polygon_df = sedona.sql("select ST_GeomFromGeoJSON(polygontable._c0) as countyshape from polygontable")
+polygon_df.show(5)

seeker/snippet/sedona_df_to_geopandas.py

@@ -0,0 +1,7 @@
+#date: 2024-09-03T16:56:27Z
+#url: https://api.github.com/gists/4e27bb8bae3bbff729866e1bc5866f8c
+#owner: https://api.github.com/users/SeanLikesData
+
+# Convert to geopandas
+df = sedona_df.toPandas()
+gdf = gpd.GeoDataFrame(df, geometry="geom")